"""Check that TensorFlow session has been initialized."""
    if tf.get_default_session() is None:
        raise RuntimeError("No default TensorFlow session found. Please call dnnlib.tflib.init_tf().")

# This work is made available under the Nvidia Source Code License-NC.
# To view a copy of this license, visit

"""Miscellaneous helper utils for Tensorflow."""

import os
import numpy as np
import tensorflow as tf

# Silence deprecation warnings from TensorFlow 1.13 onwards
import logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)
import tensorflow.contrib   # requires TensorFlow 1.x!
tf.contrib = tensorflow.contrib

from typing import Any, Iterable, List, Union

TfExpression = Union[tf.Tensor, tf.Variable, tf.Operation]
"""A type that represents a valid Tensorflow expression."""

TfExpressionEx = Union[TfExpression, int, float, np.ndarray]
"""A type that can be converted to a valid Tensorflow expression."""

def run(*args, **kwargs) -> Any:
    """Run the specified ops in the default session."""
    assert_tf_initialized()
    return tf.get_default_session().run(*args, **kwargs)

def is_tf_expression(x: Any) -> bool:
    """Check whether the input is a valid Tensorflow expression, i.e., Tensorflow Tensor, Variable, or Operation."""
    return isinstance(x, (tf.Tensor, tf.Variable, tf.Operation))

def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]:
    """Convert a Tensorflow shape to a list of ints. Retained for backwards compatibility -- use TensorShape.as_list() in new code."""
    return [dim.value for dim in shape]

def flatten(x: TfExpressionEx) -> TfExpression:
    """Shortcut function for flattening a tensor."""
    with tf.name_scope("Flatten"):
        return tf.reshape(x, [-1])

def log2(x: TfExpressionEx) -> TfExpression:
    """Logarithm in base 2."""
    with tf.name_scope("Log2"):
        return tf.log(x) * np.float32(1.0 / np.log(2.0))

def exp2(x: TfExpressionEx) -> TfExpression:
    """Exponent in base 2."""
    with tf.name_scope("Exp2"):
        return tf.exp(x * np.float32(np.log(2.0)))

def lerp(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpressionEx:
    """Linear interpolation."""
    with tf.name_scope("Lerp"):
        return a + (b - a) * t

def lerp_clip(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpression:
    """Linear interpolation with clip."""
    with tf.name_scope("LerpClip"):
        return a + (b - a) * tf.clip_by_value(t, 0.0, 1.0)

def absolute_name_scope(scope: str) -> tf.name_scope:
    """Forcefully enter the specified name scope, ignoring any surrounding scopes."""
    return tf.name_scope(scope + "/")

def absolute_variable_scope(scope: str, **kwargs) -> tf.variable_scope:
    """Forcefully enter the specified variable scope, ignoring any surrounding scopes."""
    return tf.variable_scope(tf.VariableScope(name=scope, **kwargs), auxiliary_name_scope=False)

def _sanitize_tf_config(config_dict: dict = None) -> dict:
    cfg = dict()
    cfg["rnd.np_random_seed"]               = None      # Random seed for NumPy. None = keep as is.
    cfg["rnd.tf_random_seed"]               = "auto"    # Random seed for TensorFlow. 'auto' = derive from NumPy random state. None = keep as is.
    cfg["env.TF_CPP_MIN_LOG_LEVEL"]         = "1"       # 0 = Print all available debug info from TensorFlow. 1 = Print warnings and errors, but disable debug info.
    cfg["graph_options.place_pruned_graph"] = True      # False = Check that all ops are available on the designated device. True = Skip the check for ops that are not used.
    cfg["gpu_options.allow_growth"]         = True      # False = Allocate all GPU memory at the beginning. True = Allocate only as much GPU memory as needed.

    # Remove defaults for environment variables that are already set.
    for key in list(cfg):
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            if fields[1] in os.environ:
                del cfg[key]

    # User overrides.
    if config_dict is not None:
        cfg.update(config_dict)
    return cfg

def init_tf(config_dict: dict = None) -> None:
    """Initialize TensorFlow session using good default settings."""
    if tf.get_default_session() is not None:
        return

    # Setup config dict and random seeds.
    cfg = _sanitize_tf_config(config_dict)
    np_random_seed = cfg["rnd.np_random_seed"]
    if np_random_seed is not None:
        np.random.seed(np_random_seed)
    tf_random_seed = cfg["rnd.tf_random_seed"]
    if tf_random_seed == "auto":
        tf_random_seed = np.random.randint(1 << 31)
    if tf_random_seed is not None:
        tf.set_random_seed(tf_random_seed)

    # Setup environment variables.
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            os.environ[fields[1]] = str(value)

    # Create default TensorFlow session.
    sess = create_session(cfg, force_as_default=True)
    return sess

def assert_tf_initialized():

def create_session(config_dict: dict = None, force_as_default: bool = False) -> tf.Session:
    """Create tf.Session based on config dict."""
    # Setup TensorFlow config proto.
    cfg = _sanitize_tf_config(config_dict)
    config_proto = tf.ConfigProto()
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] not in ["rnd", "env"]:
            obj = config_proto
            for field in fields[:-1]:
                obj = getattr(obj, field)
            setattr(obj, fields[-1], value)

    # Create session.
    session = tf.Session(config=config_proto)
    if force_as_default:
        # pylint: disable=protected-access
        session._default_session = session.as_default()
        session._default_session.enforce_nesting = False
        session._default_session.__enter__()
    return session

def init_uninitialized_vars(target_vars: List[tf.Variable] = None) -> None:
    """Initialize all tf.Variables that have not already been initialized.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tf.variables_initializer(tf.report_uninitialized_variables()).run()
    """
    assert_tf_initialized()
    if target_vars is None:
        target_vars = tf.global_variables()

    test_vars = []
    test_ops = []

    with tf.control_dependencies(None):  # ignore surrounding control_dependencies
        for var in target_vars:
            assert is_tf_expression(var)

            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/IsVariableInitialized:0"))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)

                with absolute_name_scope(var.name.split(":")[0]):
                    test_ops.append(tf.is_variable_initialized(var))

    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])

def set_vars(var_to_value_dict: dict) -> None:
    """Set the values of given tf.Variables.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tflib.run([tf.assign(var, value) for var, value in var_to_value_dict.items()]
    """
    assert_tf_initialized()
    ops = []
    feed_dict = {}

    for var, value in var_to_value_dict.items():
        assert is_tf_expression(var)

        try:
            setter = tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/setter:0"))  # look for existing op
        except KeyError:
            with absolute_name_scope(var.name.split(":")[0]):
                with tf.control_dependencies(None):  # ignore surrounding control_dependencies
                    setter = tf.assign(var, tf.placeholder(var.dtype, var.shape, "new_value"), name="setter")  # create new setter

        ops.append(setter)
        feed_dict[setter.op.inputs[1]] = value

    run(ops, feed_dict)

def create_var_with_large_initial_value(initial_value: np.ndarray, *args, **kwargs):
    """Create tf.Variable with large initial value without bloating the tf graph."""
    assert_tf_initialized()
    assert isinstance(initial_value, np.ndarray)
    zeros = tf.zeros(initial_value.shape, initial_value.dtype)
    var = tf.Variable(zeros, *args, **kwargs)
    set_vars({var: initial_value})
    return var

def convert_images_from_uint8(images, drange=[-1,1], nhwc_to_nchw=False):
    """Convert a minibatch of images from uint8 to float32 with configurable dynamic range.
    Can be used as an input transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if nhwc_to_nchw:
        images = tf.transpose(images, [0, 3, 1, 2])
    return images * ((drange[1] - drange[0]) / 255) + drange[0]

def convert_images_to_uint8(images, drange=[-1,1], nchw_to_nhwc=False, shrink=1, uint8_cast=True):
    """Convert a minibatch of images from float32 to uint8 with configurable dynamic range.
    Can be used as an output transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if shrink > 1:
        ksize = [1, 1, shrink, shrink]
        images = tf.nn.avg_pool(images, ksize=ksize, strides=ksize, padding="VALID", data_format="NCHW")
    if nchw_to_nhwc:
        images = tf.transpose(images, [0, 2, 3, 1])
    scale = 255 / (drange[1] - drange[0])
    images = images * scale + (0.5 - drange[0] * scale)
    if uint8_cast:
        images = tf.saturate_cast(images, tf.uint8)
    return images


t_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]

# This work is made available under the Nvidia Source Code License-NC.
# To view a copy of this license, visit

"""Miscellaneous helper utils for Tensorflow."""

import os
import numpy as np
import tensorflow as tf

# Silence deprecation warnings from TensorFlow 1.13 onwards
import logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)
import tensorflow.contrib   # requires TensorFlow 1.x!
tf.contrib = tensorflow.contrib

from typing import Any, Iterable, List, Union

TfExpression = Union[tf.Tensor, tf.Variable, tf.Operation]
"""A type that represents a valid Tensorflow expression."""

TfExpressionEx = Union[TfExpression, int, float, np.ndarray]
"""A type that can be converted to a valid Tensorflow expression."""

def run(*args, **kwargs) -> Any:
    """Run the specified ops in the default session."""
    assert_tf_initialized()
    return tf.get_default_session().run(*args, **kwargs)

def is_tf_expression(x: Any) -> bool:
    """Check whether the input is a valid Tensorflow expression, i.e., Tensorflow Tensor, Variable, or Operation."""
    return isinstance(x, (tf.Tensor, tf.Variable, tf.Operation))

def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]:
    """Convert a Tensorflow shape to a list of ints. Retained for backwards compatibility -- use TensorShape.as_list() in new code."""
    return [dim.value for dim in shape]

def flatten(x: TfExpressionEx) -> TfExpression:
    """Shortcut function for flattening a tensor."""
    with tf.name_scope("Flatten"):
        return tf.reshape(x, [-1])

def log2(x: TfExpressionEx) -> TfExpression:
    """Logarithm in base 2."""
    with tf.name_scope("Log2"):
        return tf.log(x) * np.float32(1.0 / np.log(2.0))

def exp2(x: TfExpressionEx) -> TfExpression:
    """Exponent in base 2."""
    with tf.name_scope("Exp2"):
        return tf.exp(x * np.float32(np.log(2.0)))

def lerp(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpressionEx:
    """Linear interpolation."""
    with tf.name_scope("Lerp"):
        return a + (b - a) * t

def lerp_clip(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpression:
    """Linear interpolation with clip."""
    with tf.name_scope("LerpClip"):
        return a + (b - a) * tf.clip_by_value(t, 0.0, 1.0)

def absolute_name_scope(scope: str) -> tf.name_scope:
    """Forcefully enter the specified name scope, ignoring any surrounding scopes."""
    return tf.name_scope(scope + "/")

def absolute_variable_scope(scope: str, **kwargs) -> tf.variable_scope:
    """Forcefully enter the specified variable scope, ignoring any surrounding scopes."""
    return tf.variable_scope(tf.VariableScope(name=scope, **kwargs), auxiliary_name_scope=False)

def _sanitize_tf_config(config_dict: dict = None) -> dict:
    cfg = dict()
    cfg["rnd.np_random_seed"]               = None      # Random seed for NumPy. None = keep as is.
    cfg["rnd.tf_random_seed"]               = "auto"    # Random seed for TensorFlow. 'auto' = derive from NumPy random state. None = keep as is.
    cfg["env.TF_CPP_MIN_LOG_LEVEL"]         = "1"       # 0 = Print all available debug info from TensorFlow. 1 = Print warnings and errors, but disable debug info.
    cfg["graph_options.place_pruned_graph"] = True      # False = Check that all ops are available on the designated device. True = Skip the check for ops that are not used.
    cfg["gpu_options.allow_growth"]         = True      # False = Allocate all GPU memory at the beginning. True = Allocate only as much GPU memory as needed.

    # Remove defaults for environment variables that are already set.
    for key in list(cfg):
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            if fields[1] in os.environ:
                del cfg[key]

    # User overrides.
    if config_dict is not None:
        cfg.update(config_dict)
    return cfg

def init_tf(config_dict: dict = None) -> None:
    """Initialize TensorFlow session using good default settings."""
    if tf.get_default_session() is not None:
        return

    # Setup config dict and random seeds.
    cfg = _sanitize_tf_config(config_dict)
    np_random_seed = cfg["rnd.np_random_seed"]
    if np_random_seed is not None:
        np.random.seed(np_random_seed)
    tf_random_seed = cfg["rnd.tf_random_seed"]
    if tf_random_seed == "auto":
        tf_random_seed = np.random.randint(1 << 31)
    if tf_random_seed is not None:
        tf.set_random_seed(tf_random_seed)

    # Setup environment variables.
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            os.environ[fields[1]] = str(value)

    # Create default TensorFlow session.
    sess = create_session(cfg, force_as_default=True)
    return sess

def assert_tf_initialized():
    """Check that TensorFlow session has been initialized."""
    if tf.get_default_session() is None:
        raise RuntimeError("No default TensorFlow session found. Please call dnnlib.tflib.init_tf().")

def create_session(config_dict: dict = None, force_as_default: bool = False) -> tf.Session:
    """Create tf.Session based on config dict."""
    # Setup TensorFlow config proto.
    cfg = _sanitize_tf_config(config_dict)
    config_proto = tf.ConfigProto()
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] not in ["rnd", "env"]:
            obj = config_proto
            for field in fields[:-1]:
                obj = getattr(obj, field)
            setattr(obj, fields[-1], value)

    # Create session.
    session = tf.Session(config=config_proto)
    if force_as_default:
        # pylint: disable=protected-access
        session._default_session = session.as_default()
        session._default_session.enforce_nesting = False
        session._default_session.__enter__()
    return session

def init_uninitialized_vars(target_vars: List[tf.Variable] = None) -> None:
    """Initialize all tf.Variables that have not already been initialized.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tf.variables_initializer(tf.report_uninitialized_variables()).run()
    """
    assert_tf_initialized()
    if target_vars is None:
        target_vars = tf.global_variables()

    test_vars = []
    test_ops = []

    with tf.control_dependencies(None):  # ignore surrounding control_dependencies
        for var in target_vars:
            assert is_tf_expression(var)

            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/IsVariableInitialized:0"))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)

                with absolute_name_scope(var.name.split(":")[0]):
                    test_ops.append(tf.is_variable_initialized(var))

    ini

    run([var.initializer for var in init_vars])

def set_vars(var_to_value_dict: dict) -> None:
    """Set the values of given tf.Variables.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tflib.run([tf.assign(var, value) for var, value in var_to_value_dict.items()]
    """
    assert_tf_initialized()
    ops = []
    feed_dict = {}

    for var, value in var_to_value_dict.items():
        assert is_tf_expression(var)

        try:
            setter = tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/setter:0"))  # look for existing op
        except KeyError:
            with absolute_name_scope(var.name.split(":")[0]):
                with tf.control_dependencies(None):  # ignore surrounding control_dependencies
                    setter = tf.assign(var, tf.placeholder(var.dtype, var.shape, "new_value"), name="setter")  # create new setter

        ops.append(setter)
        feed_dict[setter.op.inputs[1]] = value

    run(ops, feed_dict)

def create_var_with_large_initial_value(initial_value: np.ndarray, *args, **kwargs):
    """Create tf.Variable with large initial value without bloating the tf graph."""
    assert_tf_initialized()
    assert isinstance(initial_value, np.ndarray)
    zeros = tf.zeros(initial_value.shape, initial_value.dtype)
    var = tf.Variable(zeros, *args, **kwargs)
    set_vars({var: initial_value})
    return var

def convert_images_from_uint8(images, drange=[-1,1], nhwc_to_nchw=False):
    """Convert a minibatch of images from uint8 to float32 with configurable dynamic range.
    Can be used as an input transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if nhwc_to_nchw:
        images = tf.transpose(images, [0, 3, 1, 2])
    return images * ((drange[1] - drange[0]) / 255) + drange[0]

def convert_images_to_uint8(images, drange=[-1,1], nchw_to_nhwc=False, shrink=1, uint8_cast=True):
    """Convert a minibatch of images from float32 to uint8 with configurable dynamic range.
    Can be used as an output transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if shrink > 1:
        ksize = [1, 1, shrink, shrink]
        images = tf.nn.avg_pool(images, ksize=ksize, strides=ksize, padding="VALID", data_format="NCHW")
    if nchw_to_nhwc:
        images = tf.transpose(images, [0, 2, 3, 1])
    scale = 255 / (drange[1] - drange[0])
    images = images * scale + (0.5 - drange[0] * scale)
    if uint8_cast:
        images = tf.saturate_cast(images, tf.uint8)
    return images


# This work is made available under the Nvidia Source Code License-NC.
# To view a copy of this license, visit

"""Miscellaneous helper utils for Tensorflow."""

import os
import numpy as np
import tensorflow as tf

# Silence deprecation warnings from TensorFlow 1.13 onwards
import logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)
import tensorflow.contrib   # requires TensorFlow 1.x!
tf.contrib = tensorflow.contrib

from typing import Any, Iterable, List, Union

TfExpression = Union[tf.Tensor, tf.Variable, tf.Operation]
"""A type that represents a valid Tensorflow expression."""

TfExpressionEx = Union[TfExpression, int, float, np.ndarray]
"""A type that can be converted to a valid Tensorflow expression."""

def run(*args, **kwargs) -> Any:
    """Run the specified ops in the default session."""
    assert_tf_initialized()
    return tf.get_default_session().run(*args, **kwargs)

def is_tf_expression(x: Any) -> bool:
    """Check whether the input is a valid Tensorflow expression, i.e., Tensorflow Tensor, Variable, or Operation."""
    return isinstance(x, (tf.Tensor, tf.Variable, tf.Operation))

def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]:
    """Convert a Tensorflow shape to a list of ints. Retained for backwards compatibility -- use TensorShape.as_list() in new code."""

def flatten(x: TfExpressionEx) -> TfExpression:
    """Shortcut function for flattening a tensor."""
    with tf.name_scope("Flatten"):
        return tf.reshape(x, [-1])

def log2(x: TfExpressionEx) -> TfExpression:
    """Logarithm in base 2."""
    with tf.name_scope("Log2"):
        return tf.log(x) * np.float32(1.0 / np.log(2.0))

def exp2(x: TfExpressionEx) -> TfExpression:
    """Exponent in base 2."""
    with tf.name_scope("Exp2"):
        return tf.exp(x * np.float32(np.log(2.0)))

def lerp(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpressionEx:
    """Linear interpolation."""
    with tf.name_scope("Lerp"):
        return a + (b - a) * t

def lerp_clip(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpression:
    """Linear interpolation with clip."""
    with tf.name_scope("LerpClip"):
        return a + (b - a) * tf.clip_by_value(t, 0.0, 1.0)

def absolute_name_scope(scope: str) -> tf.name_scope:
    """Forcefully enter the specified name scope, ignoring any surrounding scopes."""
    return tf.name_scope(scope + "/")

def absolute_variable_scope(scope: str, **kwargs) -> tf.variable_scope:
    """Forcefully enter the specified variable scope, ignoring any surrounding scopes."""
    return tf.variable_scope(tf.VariableScope(name=scope, **kwargs), auxiliary_name_scope=False)

def _sanitize_tf_config(config_dict: dict = None) -> dict:
    cfg = dict()
    cfg["rnd.np_random_seed"]               = None      # Random seed for NumPy. None = keep as is.
    cfg["rnd.tf_random_seed"]               = "auto"    # Random seed for TensorFlow. 'auto' = derive from NumPy random state. None = keep as is.
    cfg["env.TF_CPP_MIN_LOG_LEVEL"]         = "1"       # 0 = Print all available debug info from TensorFlow. 1 = Print warnings and errors, but disable debug info.
    cfg["graph_options.place_pruned_graph"] = True      # False = Check that all ops are available on the designated device. True = Skip the check for ops that are not used.
    cfg["gpu_options.allow_growth"]         = True      # False = Allocate all GPU memory at the beginning. True = Allocate only as much GPU memory as needed.

    # Remove defaults for environment variables that are already set.
    for key in list(cfg):
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            if fields[1] in os.environ:
                del cfg[key]

    # User overrides.
    if config_dict is not None:
        cfg.update(config_dict)
    return cfg

def init_tf(config_dict: dict = None) -> None:
    """Initialize TensorFlow session using good default settings."""
    if tf.get_default_session() is not None:
        return

    # Setup config dict and random seeds.
    cfg = _sanitize_tf_config(config_dict)
    np_random_seed = cfg["rnd.np_random_seed"]
    if np_random_seed is not None:
        np.random.seed(np_random_seed)
    tf_random_seed = cfg["rnd.tf_random_seed"]
    if tf_random_seed == "auto":
        tf_random_seed = np.random.randint(1 << 31)
    if tf_random_seed is not None:
        tf.set_random_seed(tf_random_seed)

    # Setup environment variables.
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            os.environ[fields[1]] = str(value)

    # Create default TensorFlow session.
    sess = create_session(cfg, force_as_default=True)
    return sess

def assert_tf_initialized():
    """Check that TensorFlow session has been initialized."""
    if tf.get_default_session() is None:
        raise RuntimeError("No default TensorFlow session found. Please call dnnlib.tflib.init_tf().")

def create_session(config_dict: dict = None, force_as_default: bool = False) -> tf.Session:
    """Create tf.Session based on config dict."""
    # Setup TensorFlow config proto.
    cfg = _sanitize_tf_config(config_dict)
    config_proto = tf.ConfigProto()
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] not in ["rnd", "env"]:
            obj = config_proto
            for field in fields[:-1]:
                obj = getattr(obj, field)
            setattr(obj, fields[-1], value)

    # Create session.
    session = tf.Session(config=config_proto)
    if force_as_default:
        # pylint: disable=protected-access
        session._default_session = session.as_default()
        session._default_session.enforce_nesting = False
        session._default_session.__enter__()
    return session

def init_uninitialized_vars(target_vars: List[tf.Variable] = None) -> None:
    """Initialize all tf.Variables that have not already been initialized.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tf.variables_initializer(tf.report_uninitialized_variables()).run()
    """
    assert_tf_initialized()
    if target_vars is None:
        target_vars = tf.global_variables()

    test_vars = []
    test_ops = []

    with tf.control_dependencies(None):  # ignore surrounding control_dependencies
        for var in target_vars:
            assert is_tf_expression(var)

            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/IsVariableInitialized:0"))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)

                with absolute_name_scope(var.name.split(":")[0]):
                    test_ops.append(tf.is_variable_initialized(var))

    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])

def set_vars(var_to_value_dict: dict) -> None:
    """Set the values of given tf.Variables.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tflib.run([tf.assign(var, value) for var, value in var_to_value_dict.items()]
    """
    assert_tf_initialized()
    ops = []
    feed_dict = {}

    for var, value in var_to_value_dict.items():
        assert is_tf_expression(var)

        try:
            setter = tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/setter:0"))  # look for existing op
        except KeyError:
            with absolute_name_scope(var.name.split(":")[0]):
                with tf.control_dependencies(None):  # ignore surrounding control_dependencies
                    setter = tf.assign(var, tf.placeholder(var.dtype, var.shape, "new_value"), name="setter")  # create new setter

        ops.append(setter)
        feed_dict[setter.op.inputs[1]] = value

    run(ops, feed_dict)

def create_var_with_large_initial_value(initial_value: np.ndarray, *args, **kwargs):
    """Create tf.Variable with large initial value without bloating the tf graph."""
    assert_tf_initialized()
    assert isinstance(initial_value, np.ndarray)
    zeros = tf.zeros(initial_value.shape, initial_value.dtype)
    var = tf.Variable(zeros, *args, **kwargs)
    set_vars({var: initial_value})
    return var

def convert_images_from_uint8(images, drange=[-1,1], nhwc_to_nchw=False):
    """Convert a minibatch of images from uint8 to float32 with configurable dynamic range.
    Can be used as an input transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if nhwc_to_nchw:
        images = tf.transpose(images, [0, 3, 1, 2])
    return images * ((drange[1] - drange[0]) / 255) + drange[0]

def convert_images_to_uint8(images, drange=[-1,1], nchw_to_nhwc=False, shrink=1, uint8_cast=True):
    """Convert a minibatch of images from float32 to uint8 with configurable dynamic range.
    Can be used as an output transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if shrink > 1:
        ksize = [1, 1, shrink, shrink]
        images = tf.nn.avg_pool(images, ksize=ksize, strides=ksize, padding="VALID", data_format="NCHW")
    if nchw_to_nhwc:
        images = tf.transpose(images, [0, 2, 3, 1])
    scale = 255 / (drange[1] - drange[0])
    images = images * scale + (0.5 - drange[0] * scale)
    if uint8_cast:
        images = tf.saturate_cast(images, tf.uint8)
    return images


try:
            setter = tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/setter:0"))  # look for existing op
        except KeyError:
            with absolute_name_scope(var.name.split(":")[0]):
                with tf.control_dependencies(None):  # ignore surrounding control_dependencies
                    setter = tf.assign(var, tf.placeholder(var.dtype, var.shape, "new_value"), name="setter")

# This work is made available under the Nvidia Source Code License-NC.
# To view a copy of this license, visit

"""Miscellaneous helper utils for Tensorflow."""

import os
import numpy as np
import tensorflow as tf

# Silence deprecation warnings from TensorFlow 1.13 onwards
import logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)
import tensorflow.contrib   # requires TensorFlow 1.x!
tf.contrib = tensorflow.contrib

from typing import Any, Iterable, List, Union

TfExpression = Union[tf.Tensor, tf.Variable, tf.Operation]
"""A type that represents a valid Tensorflow expression."""

TfExpressionEx = Union[TfExpression, int, float, np.ndarray]
"""A type that can be converted to a valid Tensorflow expression."""

def run(*args, **kwargs) -> Any:
    """Run the specified ops in the default session."""
    assert_tf_initialized()
    return tf.get_default_session().run(*args, **kwargs)

def is_tf_expression(x: Any) -> bool:
    """Check whether the input is a valid Tensorflow expression, i.e., Tensorflow Tensor, Variable, or Operation."""
    return isinstance(x, (tf.Tensor, tf.Variable, tf.Operation))

def shape_to_list(shape: Iterable[tf.Dimension]) -> List[Union[int, None]]:
    """Convert a Tensorflow shape to a list of ints. Retained for backwards compatibility -- use TensorShape.as_list() in new code."""
    return [dim.value for dim in shape]

def flatten(x: TfExpressionEx) -> TfExpression:
    """Shortcut function for flattening a tensor."""
    with tf.name_scope("Flatten"):
        return tf.reshape(x, [-1])

def log2(x: TfExpressionEx) -> TfExpression:
    """Logarithm in base 2."""
    with tf.name_scope("Log2"):
        return tf.log(x) * np.float32(1.0 / np.log(2.0))

def exp2(x: TfExpressionEx) -> TfExpression:
    """Exponent in base 2."""
    with tf.name_scope("Exp2"):
        return tf.exp(x * np.float32(np.log(2.0)))

def lerp(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpressionEx:
    """Linear interpolation."""
    with tf.name_scope("Lerp"):
        return a + (b - a) * t

def lerp_clip(a: TfExpressionEx, b: TfExpressionEx, t: TfExpressionEx) -> TfExpression:
    """Linear interpolation with clip."""
    with tf.name_scope("LerpClip"):
        return a + (b - a) * tf.clip_by_value(t, 0.0, 1.0)

def absolute_name_scope(scope: str) -> tf.name_scope:
    """Forcefully enter the specified name scope, ignoring any surrounding scopes."""
    return tf.name_scope(scope + "/")

def absolute_variable_scope(scope: str, **kwargs) -> tf.variable_scope:
    """Forcefully enter the specified variable scope, ignoring any surrounding scopes."""
    return tf.variable_scope(tf.VariableScope(name=scope, **kwargs), auxiliary_name_scope=False)

def _sanitize_tf_config(config_dict: dict = None) -> dict:
    cfg = dict()
    cfg["rnd.np_random_seed"]               = None      # Random seed for NumPy. None = keep as is.
    cfg["rnd.tf_random_seed"]               = "auto"    # Random seed for TensorFlow. 'auto' = derive from NumPy random state. None = keep as is.
    cfg["env.TF_CPP_MIN_LOG_LEVEL"]         = "1"       # 0 = Print all available debug info from TensorFlow. 1 = Print warnings and errors, but disable debug info.
    cfg["graph_options.place_pruned_graph"] = True      # False = Check that all ops are available on the designated device. True = Skip the check for ops that are not used.
    cfg["gpu_options.allow_growth"]         = True      # False = Allocate all GPU memory at the beginning. True = Allocate only as much GPU memory as needed.

    # Remove defaults for environment variables that are already set.
    for key in list(cfg):
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            if fields[1] in os.environ:
                del cfg[key]

    # User overrides.
    if config_dict is not None:
        cfg.update(config_dict)
    return cfg

def init_tf(config_dict: dict = None) -> None:
    """Initialize TensorFlow session using good default settings."""
    if tf.get_default_session() is not None:
        return

    # Setup config dict and random seeds.
    cfg = _sanitize_tf_config(config_dict)
    np_random_seed = cfg["rnd.np_random_seed"]
    if np_random_seed is not None:
        np.random.seed(np_random_seed)
    tf_random_seed = cfg["rnd.tf_random_seed"]
    if tf_random_seed == "auto":
        tf_random_seed = np.random.randint(1 << 31)
    if tf_random_seed is not None:
        tf.set_random_seed(tf_random_seed)

    # Setup environment variables.
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] == "env":
            assert len(fields) == 2
            os.environ[fields[1]] = str(value)

    # Create default TensorFlow session.
    sess = create_session(cfg, force_as_default=True)
    return sess

def assert_tf_initialized():
    """Check that TensorFlow session has been initialized."""
    if tf.get_default_session() is None:
        raise RuntimeError("No default TensorFlow session found. Please call dnnlib.tflib.init_tf().")

def create_session(config_dict: dict = None, force_as_default: bool = False) -> tf.Session:
    """Create tf.Session based on config dict."""
    # Setup TensorFlow config proto.
    cfg = _sanitize_tf_config(config_dict)
    config_proto = tf.ConfigProto()
    for key, value in cfg.items():
        fields = key.split(".")
        if fields[0] not in ["rnd", "env"]:
            obj = config_proto
            for field in fields[:-1]:
                obj = getattr(obj, field)
            setattr(obj, fields[-1], value)

    # Create session.
    session = tf.Session(config=config_proto)
    if force_as_default:
        # pylint: disable=protected-access
        session._default_session = session.as_default()
        session._default_session.enforce_nesting = False
        session._default_session.__enter__()
    return session

def init_uninitialized_vars(target_vars: List[tf.Variable] = None) -> None:
    """Initialize all tf.Variables that have not already been initialized.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tf.variables_initializer(tf.report_uninitialized_variables()).run()
    """
    assert_tf_initialized()
    if target_vars is None:
        target_vars = tf.global_variables()

    test_vars = []
    test_ops = []

    with tf.control_dependencies(None):  # ignore surrounding control_dependencies
        for var in target_vars:
            assert is_tf_expression(var)

            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/IsVariableInitialized:0"))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)

                with absolute_name_scope(var.name.split(":")[0]):
                    test_ops.append(tf.is_variable_initialized(var))

    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])

def set_vars(var_to_value_dict: dict) -> None:
    """Set the values of given tf.Variables.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tflib.run([tf.assign(var, value) for var, value in var_to_value_dict.items()]
    """
    assert_tf_initialized()
    ops = []
    feed_dict = {}

    for var, value in var_to_value_dict.items():
        assert is_tf_expression(var)

  # create new setter

        ops.append(setter)
        feed_dict[setter.op.inputs[1]] = value

    run(ops, feed_dict)

def create_var_with_large_initial_value(initial_value: np.ndarray, *args, **kwargs):
    """Create tf.Variable with large initial value without bloating the tf graph."""
    assert_tf_initialized()
    assert isinstance(initial_value, np.ndarray)
    zeros = tf.zeros(initial_value.shape, initial_value.dtype)
    var = tf.Variable(zeros, *args, **kwargs)
    set_vars({var: initial_value})
    return var

def convert_images_from_uint8(images, drange=[-1,1], nhwc_to_nchw=False):
    """Convert a minibatch of images from uint8 to float32 with configurable dynamic range.
    Can be used as an input transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if nhwc_to_nchw:
        images = tf.transpose(images, [0, 3, 1, 2])
    return images * ((drange[1] - drange[0]) / 255) + drange[0]

def convert_images_to_uint8(images, drange=[-1,1], nchw_to_nhwc=False, shrink=1, uint8_cast=True):
    """Convert a minibatch of images from float32 to uint8 with configurable dynamic range.
    Can be used as an output transformation for Network.run().
    """
    images = tf.cast(images, tf.float32)
    if shrink > 1:
        ksize = [1, 1, shrink, shrink]
        images = tf.nn.avg_pool(images, ksize=ksize, strides=ksize, padding="VALID", data_format="NCHW")
    if nchw_to_nhwc:
        images = tf.transpose(images, [0, 2, 3, 1])
    scale = 255 / (drange[1] - drange[0])
    images = images * scale + (0.5 - drange[0] * scale)
    if uint8_cast:
        images = tf.saturate_cast(images, tf.uint8)
    return images


from __future__ import print_function

import configargparse
import copy
import unittest
import glob
import os
import codecs

import onmt
import onmt.inputters
import onmt.opts
import onmt.bin.preprocess as preprocess

parser = configargparse.ArgumentParser(description='preprocess.py')
onmt.opts.preprocess_opts(parser)

SAVE_DATA_PREFIX = 'data/test_preprocess'

default_opts = [
    '-data_type', 'text',
    '-train_src', 'data/src-train.txt',
    '-train_tgt', 'data/tgt-train.txt',
    '-valid_src', 'data/src-val.txt',
    '-valid_tgt', 'data/tgt-val.txt',
    '-save_data', SAVE_DATA_PREFIX
]

opt = parser.parse_known_args(default_opts)[0]

class TestData(unittest.TestCase):

super(TestData, self).__init__(*args, **kwargs)
        self.opt = opt

    def dataset_build(self, opt):
        fields = onmt.inputters.get_fields("text", 0, 0)

        if hasattr(opt, 'src_vocab') and len(opt.src_vocab) > 0:
            with codecs.open(opt.src_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')
        if hasattr(opt, 'tgt_vocab') and len(opt.tgt_vocab) > 0:
            with codecs.open(opt.tgt_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')

        src_reader = onmt.inputters.str2reader[opt.data_type].from_opt(opt)
        tgt_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        align_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        preprocess.build_save_dataset(
            'train', fields, src_reader, tgt_reader, align_reader, opt)

        preprocess.build_save_dataset(
            'valid', fields, src_reader, tgt_reader, align_reader, opt)

        # Remove the generated *pt files.
        for pt in glob.glob(SAVE_DATA_PREFIX + '*.pt'):
            os.remove(pt)
        if hasattr(opt, 'src_vocab') and os.path.exists(opt.src_vocab):
            os.remove(opt.src_vocab)
        if hasattr(opt, 'tgt_vocab') and os.path.exists(opt.tgt_vocab):
            os.remove(opt.tgt_vocab)

def _add_test(param_setting, methodname):
    """
    Adds a Test to TestData according to settings

    Args:
        param_setting: list of tuples of (param, setting)
        methodname: name of the method that gets called
    """

    def test_method(self):
        if param_setting:
            opt = copy.deepcopy(self.opt)
            for param, setting in param_setting:
                setattr(opt, param, setting)
        else:
            opt = self.opt
        getattr(self, methodname)(opt)
    if param_setting:
        name = 'test_' + methodname + "_" + "_".join(
            str(param_setting).split())
    else:
        name = 'test_' + methodname + '_standard'
    setattr(TestData, name, test_method)
    test_method.__name__ = name

test_databuild = [[],
                  [('src_vocab_size', 1),
                   ('tgt_vocab_size', 1)],
                  [('src_vocab_size', 10000),
                   ('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 1)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('dynamic_dict', True)],
                  [('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('shard_size', 500000)],
                  [('src_vocab', '/tmp/src_vocab.txt'),
                   ('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]

for p in test_databuild:
    _add_test(p, 'dataset_build')

# Test image preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'img'),
                         ('src_dir', '/tmp/im2text/images'),
                         ('train_src', ['/tmp/im2text/src-train-head.txt']),
                         ('train_tgt', ['/tmp/im2text/tgt-train-head.txt']),
                         ('valid_src', '/tmp/im2text/src-val-head.txt'),
                         ('valid_tgt', '/tmp/im2text/tgt-val-head.txt'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')

# Test audio preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 3200)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'audio'),
                         ('src_dir', '/tmp/speech/an4_dataset'),
                         ('train_src', ['/tmp/speech/src-train-head.txt']),
                         ('train_tgt', ['/tmp/speech/tgt-train-head.txt']),
                         ('valid_src', '/tmp/speech/src-val-head.txt'),
                         ('valid_tgt', '/tmp/speech/tgt-val-head.txt'),
                         ('sample_rate', 16000),
                         ('window_size', 0.04),
                         ('window_stride', 0.02),
                         ('window', 'hamming'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')


super(TestData, self).__init__(*args, **kwargs)
        self.opt = opt

from __future__ import print_function

import configargparse
import copy
import unittest
import glob
import os
import codecs

import onmt
import onmt.inputters
import onmt.opts
import onmt.bin.preprocess as preprocess

parser = configargparse.ArgumentParser(description='preprocess.py')
onmt.opts.preprocess_opts(parser)

SAVE_DATA_PREFIX = 'data/test_preprocess'

default_opts = [
    '-data_type', 'text',
    '-train_src', 'data/src-train.txt',
    '-train_tgt', 'data/tgt-train.txt',
    '-valid_src', 'data/src-val.txt',
    '-valid_tgt', 'data/tgt-val.txt',
    '-save_data', SAVE_DATA_PREFIX
]

opt = parser.parse_known_args(default_opts)[0]

class TestData(unittest.TestCase):
    def __init__(self, *args, **kwargs):

    def dataset_build(self, opt):
        fields = onmt.inputters.get_fields("text", 0, 0)

        if hasattr(opt, 'src_vocab') and len(opt.src_vocab) > 0:
            with codecs.open(opt.src_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')
        if hasattr(opt, 'tgt_vocab') and len(opt.tgt_vocab) > 0:
            with codecs.open(opt.tgt_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')

        src_reader = onmt.inputters.str2reader[opt.data_type].from_opt(opt)
        tgt_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        align_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        preprocess.build_save_dataset(
            'train', fields, src_reader, tgt_reader, align_reader, opt)

        preprocess.build_save_dataset(
            'valid', fields, src_reader, tgt_reader, align_reader, opt)

        # Remove the generated *pt files.
        for pt in glob.glob(SAVE_DATA_PREFIX + '*.pt'):
            os.remove(pt)
        if hasattr(opt, 'src_vocab') and os.path.exists(opt.src_vocab):
            os.remove(opt.src_vocab)
        if hasattr(opt, 'tgt_vocab') and os.path.exists(opt.tgt_vocab):
            os.remove(opt.tgt_vocab)

def _add_test(param_setting, methodname):
    """
    Adds a Test to TestData according to settings

    Args:
        param_setting: list of tuples of (param, setting)
        methodname: name of the method that gets called
    """

    def test_method(self):
        if param_setting:
            opt = copy.deepcopy(self.opt)
            for param, setting in param_setting:
                setattr(opt, param, setting)
        else:
            opt = self.opt
        getattr(self, methodname)(opt)
    if param_setting:
        name = 'test_' + methodname + "_" + "_".join(
            str(param_setting).split())
    else:
        name = 'test_' + methodname + '_standard'
    setattr(TestData, name, test_method)
    test_method.__name__ = name

test_databuild = [[],
                  [('src_vocab_size', 1),
                   ('tgt_vocab_size', 1)],
                  [('src_vocab_size', 10000),
                   ('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 1)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('dynamic_dict', True)],
                  [('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('shard_size', 500000)],
                  [('src_vocab', '/tmp/src_vocab.txt'),
                   ('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]

for p in test_databuild:
    _add_test(p, 'dataset_build')

# Test image preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'img'),
                         ('src_dir', '/tmp/im2text/images'),
                         ('train_src', ['/tmp/im2text/src-train-head.txt']),
                         ('train_tgt', ['/tmp/im2text/tgt-train-head.txt']),
                         ('valid_src', '/tmp/im2text/src-val-head.txt'),
                         ('valid_tgt', '/tmp/im2text/tgt-val-head.txt'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')

# Test audio preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 3200)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'audio'),
                         ('src_dir', '/tmp/speech/an4_dataset'),
                         ('train_src', ['/tmp/speech/src-train-head.txt']),
                         ('train_tgt', ['/tmp/speech/tgt-train-head.txt']),
                         ('valid_src', '/tmp/speech/src-val-head.txt'),
                         ('valid_tgt', '/tmp/speech/tgt-val-head.txt'),
                         ('sample_rate', 16000),
                         ('window_size', 0.04),
                         ('window_stride', 0.02),
                         ('window', 'hamming'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')


from __future__ import print_function

import configargparse
import copy
import unittest
import glob
import os
import codecs

import onmt
import onmt.inputters
import onmt.opts
import onmt.bin.preprocess as preprocess

parser = configargparse.ArgumentParser(description='preprocess.py')
onmt.opts.preprocess_opts(parser)

SAVE_DATA_PREFIX = 'data/test_preprocess'

default_opts = [
    '-data_type', 'text',
    '-train_src', 'data/src-train.txt',
    '-train_tgt', 'data/tgt-train.txt',
    '-valid_src', 'data/src-val.txt',
    '-valid_tgt', 'data/tgt-val.txt',
    '-save_data', SAVE_DATA_PREFIX
]

opt = parser.parse_known_args(default_opts)[0]

class TestData(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestData, self).__init__(*args, **kwargs)
        self.opt = opt

    def dataset_build(self, opt):
        fields = o

        if hasattr(opt, 'src_vocab') and len(opt.src_vocab) > 0:
            with codecs.open(opt.src_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')
        if hasattr(opt, 'tgt_vocab') and len(opt.tgt_vocab) > 0:
            with codecs.open(opt.tgt_vocab, 'w', 'utf-8') as f:
                f.write('a\nb\nc\nd\ne\nf\n')

        src_reader = onmt.inputters.str2reader[opt.data_type].from_opt(opt)
        tgt_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        align_reader = onmt.inputters.str2reader["text"].from_opt(opt)
        preprocess.build_save_dataset(
            'train', fields, src_reader, tgt_reader, align_reader, opt)

        preprocess.build_save_dataset(
            'valid', fields, src_reader, tgt_reader, align_reader, opt)

        # Remove the generated *pt files.
        for pt in glob.glob(SAVE_DATA_PREFIX + '*.pt'):
            os.remove(pt)
        if hasattr(opt, 'src_vocab') and os.path.exists(opt.src_vocab):
            os.remove(opt.src_vocab)
        if hasattr(opt, 'tgt_vocab') and os.path.exists(opt.tgt_vocab):
            os.remove(opt.tgt_vocab)

def _add_test(param_setting, methodname):
    """
    Adds a Test to TestData according to settings

    Args:
        param_setting: list of tuples of (param, setting)
        methodname: name of the method that gets called
    """

    def test_method(self):
        if param_setting:
            opt = copy.deepcopy(self.opt)
            for param, setting in param_setting:
                setattr(opt, param, setting)
        else:
            opt = self.opt
        getattr(self, methodname)(opt)
    if param_setting:
        name = 'test_' + methodname + "_" + "_".join(
            str(param_setting).split())
    else:
        name = 'test_' + methodname + '_standard'
    setattr(TestData, name, test_method)
    test_method.__name__ = name

test_databuild = [[],
                  [('src_vocab_size', 1),
                   ('tgt_vocab_size', 1)],
                  [('src_vocab_size', 10000),
                   ('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 1)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('dynamic_dict', True)],
                  [('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('share_vocab', True)],
                  [('dynamic_dict', True),
                   ('shard_size', 500000)],
                  [('src_vocab', '/tmp/src_vocab.txt'),
                   ('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]

for p in test_databuild:
    _add_test(p, 'dataset_build')

# Test image preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'img'),
                         ('src_dir', '/tmp/im2text/images'),
                         ('train_src', ['/tmp/im2text/src-train-head.txt']),
                         ('train_tgt', ['/tmp/im2text/tgt-train-head.txt']),
                         ('valid_src', '/tmp/im2text/src-val-head.txt'),
                         ('valid_tgt', '/tmp/im2text/tgt-val-head.txt'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')

# Test audio preprocessing
test_databuild = [[],
                  [('tgt_vocab_size', 1)],
                  [('tgt_vocab_size', 10000)],
                  [('src_seq_len', 1)],
                  [('src_seq_len', 5000)],
                  [('src_seq_length_trunc', 3200)],
                  [('src_seq_length_trunc', 5000)],
                  [('tgt_seq_len', 1)],
                  [('tgt_seq_len', 5000)],
                  [('tgt_seq_length_trunc', 1)],
                  [('tgt_seq_length_trunc', 5000)],
                  [('shuffle', 0)],
                  [('lower', True)],
                  [('shard_size', 5)],
                  [('shard_size', 50)],
                  [('tgt_vocab', '/tmp/tgt_vocab.txt')],
                  ]
test_databuild_common = [('data_type', 'audio'),
                         ('src_dir', '/tmp/speech/an4_dataset'),
                         ('train_src', ['/tmp/speech/src-train-head.txt']),
                         ('train_tgt', ['/tmp/speech/tgt-train-head.txt']),
                         ('valid_src', '/tmp/speech/src-val-head.txt'),
                         ('valid_tgt', '/tmp/speech/tgt-val-head.txt'),
                         ('sample_rate', 16000),
                         ('window_size', 0.04),
                         ('window_stride', 0.02),
                         ('window', 'hamming'),
                         ]
for p in test_databuild:
    _add_test(p + test_databuild_common, 'dataset_build')


import logging
import requests
import json

params = json.loads(event)
  url = params['host'] + params['path']
  headers = {
    'Authorization': 'APPCODE ' + params['appcode'],
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
  }
  r = requests.request(method=params['method'], url=url, data=params['payload'], headers=headers)
  return r.text


params = json.loads(event)
  url = params['host'] + params['path']
  headers = {
    'Authorization': 'APPCODE ' + params['appcode'],
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
  }
  r = requests.request(method=params['method'], url=url, data=params['payload'], headers=headers)
  return r.text

import logging
import requests
import json

def my_handler(event, context):


import logging
import requests
import json

def my_handler(event, context):

  params = json.loads(event)
  url = params['host'] + params['path']
  headers = {
    'Authorization': 'APPCODE ' + params['appcode'],
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
  }
  r = requests.request(method=params['method'], url=url, data=params['payload'], headers=headers)


# Test for one implementation of the interface
import re
from unittest import TestCase

from lexicon.tests.providers import integration_tests

# Hook into testing framework by inheriting unittest.TestCase and reuse
# the tests which *each and every* implementation of the interface must
class MythicBeastsProviderTests(TestCase, integration_tests.IntegrationTestsV2):
    """Integration tests for Mythic Beasts provider"""

    provider_name = "mythicbeasts"
    domain = "lexitus.co.uk"

    def _filter_post_data_parameters(self):
        return ["access_token"]

return ["Authorization"]

    def _filter_response(self, response):
        """See `IntegrationTests._filter_response` for more information on how
        to filter the provider response."""

        if "string" in response["body"]:
            response["body"]["string"] = re.sub(
                br"\"access_token\":\"[\w-]+\"",
                b'"access_token": "DUMMY_TOKEN"',
                response["body"]["string"],
            )

        return response

    def _test_fallback_fn(self):
        return lambda x: "placeholder_" + x if x not in ("auth_token") else ""


return lambda x: "placeholder_" + x if x not in ("auth_token") else ""

# Test for one implementation of the interface
import re
from unittest import TestCase

from lexicon.tests.providers import integration_tests

# Hook into testing framework by inheriting unittest.TestCase and reuse
# the tests which *each and every* implementation of the interface must
class MythicBeastsProviderTests(TestCase, integration_tests.IntegrationTestsV2):
    """Integration tests for Mythic Beasts provider"""

    provider_name = "mythicbeasts"
    domain = "lexitus.co.uk"

    def _filter_post_data_parameters(self):
        return ["access_token"]

    def _filter_headers(self):
        return ["Authorization"]

    def _filter_response(self, response):
        """See `IntegrationTests._filter_response` for more information on how
        to filter the provider response."""

        if "string" in response["body"]:
            response["body"]["string"] = re.sub(
                br"\"access_token\":\"[\w-]+\"",
                b'"access_token": "DUMMY_TOKEN"',
                response["body"]["string"],
            )

        return response

    def _test_fallback_fn(self):


# Test for one implementation of the interface
import re
from unittest import TestCase

from lexicon.tests.providers import integration_tests

# Hook into testing framework by inheriting unittest.TestCase and reuse
# the tests which *each and every* implementation of the interface must
class MythicBeastsProviderTests(TestCase, integration_tests.IntegrationTestsV2):
    """Integration tests for Mythic Beasts provider"""

    provider_name = "mythicbeasts"
    domain = "lexitus.co.uk"

    def _filter_post_data_parameters(self):
        return ["access_token"]

    def _filter_headers(self):
        return ["Authorization"]

    def _filter_response(self, response):
        """See `IntegrationTests._filter_response` for more information on how
        to filter the provider response."""

        if "string" in response["body"]:
            response["body"]["string"] = re.sub(
                br"\"access_token\":\"[\w-]+\"",
                b'"access_token": "DUMMY_TOKEN"',
                response["body"]["string"],
            )

        return resp

    def _test_fallback_fn(self):
        return lambda x: "placeholder_" + x if x not in ("auth_token") else ""


        if "string" in response["body"]:
            response["body"]["string"] = re.sub(
                br"\"access_token\":\"[\w-]+\"",
                b'"access_token": "DUMMY_TOKEN"',
                response["body"]["string"],
            )

        return response

# Test for one implementation of the interface
import re
from unittest import TestCase

from lexicon.tests.providers import integration_tests

# Hook into testing framework by inheriting unittest.TestCase and reuse
# the tests which *each and every* implementation of the interface must
class MythicBeastsProviderTests(TestCase, integration_tests.IntegrationTestsV2):
    """Integration tests for Mythic Beasts provider"""

    provider_name = "mythicbeasts"
    domain = "lexitus.co.uk"

    def _filter_post_data_parameters(self):
        return ["access_token"]

    def _filter_headers(self):
        return ["Authorization"]

    def _filter_response(self, response):
        """See `IntegrationTests._filter_response` for more information on how
        to filter the provider response."""


import os
import re
import numpy as np
from datetime import date
import string

from pepper.brain.utils.constants import CAPITALIZED_TYPES
from pepper.language.utils.atoms import Emotion

def read_query(query_filename):
    """
    Read a query from file and return as a string
    Parameters
    ----------
    query_filename: str name of the query. It will be looked for in the queries folder of this project

    Returns
    -------
    query: str the query with placeholders for the query parameters, as a string to be formatted

    """
    with open(os.path.join(os.path.dirname(__file__), "../queries/{}.rq".format(query_filename))) as fr:
        query = fr.read()
    return query

def is_proper_noun(types):
    return any(i in types for i in CAPITALIZED_TYPES)

def casefold_text(text, format='triple'):
    if format == 'triple':
        if isinstance(text, basestring):
            for sign in string.punctuation:
                text = text.replace(sign, "-")

            text = text.lower().replace(" ", "-").strip('-')

        return re.sub('-+', '-', text)

    elif format == 'natural':
        return text.lower().replace("-", " ").strip() if isinstance(text, basestring) else text

    else:
        return text

def casefold_capsule(capsule, format='triple'):
    """
    Function for formatting a capsule into triple format or natural language format
    Parameters
    ----------
    capsule:
    format

    Returns
    -------

    """
    for k, v in capsule.items():
        if isinstance(v, dict):
            capsule[k] = casefold_capsule(v, format=format)
        else:
            capsule[k] = casefold_text(v, format=format)

    return capsule

def date_from_uri(uri):
    [year, month, day] = uri.split('/')[-1].split('-')
    return date(int(year), int(month), int(day))

def hash_claim_id(triple):
    return '_'.join(triple)

def confidence_to_certainty_value(confidence):
    if confidence is not None:
        if confidence > .90:
            return 'CERTAIN'
        elif confidence >= .50:
            return 'PROBABLE'
        elif confidence > 0:
            return 'POSSIBLE'
    return 'UNDERSPECIFIED'

def polarity_to_polarity_value(polarity):
    if polarity is not None:
        if polarity > 0:
            return 'POSITIVE'
        elif polarity < 0:
            return 'NEGATIVE'
    return 'UNDERSPECIFIED'

def sentiment_to_sentiment_value(sentiment):
    if sentiment is not None:
        if sentiment > 0:
            return 'POSITIVE'
        elif sentiment < 0:
            return 'NEGATIVE'
        elif sentiment == 0:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def emotion_to_emotion_value(emotion):
    if emotion is not None:
        if emotion == Emotion.ANGER:
            return 'ANGER'
        elif emotion == Emotion.DISGUST:
            return 'DISGUST'
        elif emotion == Emotion.FEAR:
            return 'FEAR'
        elif emotion == Emotion.JOY:
            return 'JOY'
        elif emotion == Emotion.SADNESS:
            return 'SADNESS'
        elif emotion == Emotion.SURPRISE:
            return 'SURPRISE'
        elif emotion == Emotion.NEUTRAL:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def replace_in_file(file, word, word_replacement):
    for i, line in enumerate(open(file)):
        line.replace(':%s' % word, ':%s' % word_replacement)
        line.replace('"%s' % word, '"%s' % word_replacement)

def get_object_id(memory, category):
    cat_mem = memory.get(casefold_text(category, format='triple'), {'brain_ids': [], 'local_ids': [], 'ids': []})
    l = cat_mem['ids'][:]
    id = l[0]
    tail = l[1:]

    cat_mem['ids'] = tail
    memory[casefold_text(category, format='triple')] = cat_mem
    return id, memory

return 1 / (1 + np.exp(-z * growth_rate))


"""
    Read a query from file and return as a string
    Parameters
    ----------
    query_filename: str name of the query. It will be looked for in the queries folder of this project

    Returns
    -------
    query: str the query with placeholders for the query parameters, as a string to be formatted

    """
    with open(os.path.join(os.path.dirname(__file__), "../queries/{}.rq".format(query_filename))) as fr:
        query = fr.read()
    return query

import os
import re
import numpy as np
from datetime import date
import string

from pepper.brain.utils.constants import CAPITALIZED_TYPES
from pepper.language.utils.atoms import Emotion

def read_query(query_filename):

def is_proper_noun(types):
    return any(i in types for i in CAPITALIZED_TYPES)

def casefold_text(text, format='triple'):
    if format == 'triple':
        if isinstance(text, basestring):
            for sign in string.punctuation:
                text = text.replace(sign, "-")

            text = text.lower().replace(" ", "-").strip('-')

        return re.sub('-+', '-', text)

    elif format == 'natural':
        return text.lower().replace("-", " ").strip() if isinstance(text, basestring) else text

    else:
        return text

def casefold_capsule(capsule, format='triple'):
    """
    Function for formatting a capsule into triple format or natural language format
    Parameters
    ----------
    capsule:
    format

    Returns
    -------

    """
    for k, v in capsule.items():
        if isinstance(v, dict):
            capsule[k] = casefold_capsule(v, format=format)
        else:
            capsule[k] = casefold_text(v, format=format)

    return capsule

def date_from_uri(uri):
    [year, month, day] = uri.split('/')[-1].split('-')
    return date(int(year), int(month), int(day))

def hash_claim_id(triple):
    return '_'.join(triple)

def confidence_to_certainty_value(confidence):
    if confidence is not None:
        if confidence > .90:
            return 'CERTAIN'
        elif confidence >= .50:
            return 'PROBABLE'
        elif confidence > 0:
            return 'POSSIBLE'
    return 'UNDERSPECIFIED'

def polarity_to_polarity_value(polarity):
    if polarity is not None:
        if polarity > 0:
            return 'POSITIVE'
        elif polarity < 0:
            return 'NEGATIVE'
    return 'UNDERSPECIFIED'

def sentiment_to_sentiment_value(sentiment):
    if sentiment is not None:
        if sentiment > 0:
            return 'POSITIVE'
        elif sentiment < 0:
            return 'NEGATIVE'
        elif sentiment == 0:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def emotion_to_emotion_value(emotion):
    if emotion is not None:
        if emotion == Emotion.ANGER:
            return 'ANGER'
        elif emotion == Emotion.DISGUST:
            return 'DISGUST'
        elif emotion == Emotion.FEAR:
            return 'FEAR'
        elif emotion == Emotion.JOY:
            return 'JOY'
        elif emotion == Emotion.SADNESS:
            return 'SADNESS'
        elif emotion == Emotion.SURPRISE:
            return 'SURPRISE'
        elif emotion == Emotion.NEUTRAL:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def replace_in_file(file, word, word_replacement):
    for i, line in enumerate(open(file)):
        line.replace(':%s' % word, ':%s' % word_replacement)
        line.replace('"%s' % word, '"%s' % word_replacement)

def get_object_id(memory, category):
    cat_mem = memory.get(casefold_text(category, format='triple'), {'brain_ids': [], 'local_ids': [], 'ids': []})
    l = cat_mem['ids'][:]
    id = l[0]
    tail = l[1:]

    cat_mem['ids'] = tail
    memory[casefold_text(category, format='triple')] = cat_mem
    return id, memory

def sigmoid(z, growth_rate=1):
    return 1 / (1 + np.exp(-z * growth_rate))


import os
import re
import numpy as np
from datetime import date
import string

from pepper.brain.utils.constants import CAPITALIZED_TYPES
from pepper.language.utils.atoms import Emotion

def read_query(query_filename):
    """
    Read a query from file and return as a string
    Parameters
    ----------
    query_filename: str name of the query. It will be looked for in the queries folder of this project

    Returns
    -------
    query: str the query with placeholders for the query parameters, as a string to be formatted

    """
    with open(os.path.join(os.path.dirname(__file__), "../queries/{}.rq".format(query_filename))) as fr:
        query = fr.read()
    return query

def is_proper_noun(types):
    return any(i in types for i in CAPITALIZED_TYPES)

def casefold_text(text, format='triple'):
    if format == 'triple':
        if isinstance(text, basestring):
            for sign in string.punctuation:
                text = text.replace(sign, "-")

            text = text.lower().replace(" ", "-").strip('-')

        return re.sub('-+', '-', text)

    elif format == 'natural':
        return text.lower().replace("-", " ").strip() if isinstance(text, basestring) else text

    else:
        return text

def casefold_capsule(capsule, format='triple'):
    """
    Function for formatting a capsule into triple format or natural language format
    Parameters
    ----------
    capsule:
    format

    Returns
    -------

    """
    for k, v in capsule.items():
        if isinstance(v, dict):
            capsule[k] = casefold_capsule(v, format=format)
        else:
            capsule[k] = casefold_text(v, format=format)

    return capsule

def date_from_uri(uri):
    [year, month, day] = uri.split('/')[-1].split('-')
    return date(int(year), int(month), int(day))

def hash_claim_id(triple):
    return '_'.join(triple)

def confidence_to_certainty_value(confidence):
    if confidence is not None:
        if confidence > .90:
            return 'CERTAIN'
        elif confidence >= .50:
            return 'PROBABLE'
        elif confidence > 0:
            return 'POSSIBLE'
    return 'UNDERSPECIFIED'

def polarity_to_polarity_value(polarity):
    if polarity is not None:
        if polarity > 0:
            return 'POSITIVE'
        elif polarity < 0:
            return 'NEGATIVE'
    return 'UNDERSPECIFIED'

def sentiment_to_sentiment_value(sentiment):
    if sentiment is not None:
        if sentiment > 0:
            return 'POSITIVE'
        elif sentiment < 0:
            return 'NEGATIVE'
        elif sentiment == 0:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def emotion_to_emotion_value(emotion):
    if emotion is not None:
        if emotion == Emotion.ANGER:
            return 'ANGER'
        elif emotion == Emotion.DISGUST:
            return 'DISGUST'
        elif emotion == Emotion.FEAR:
            return 'FEAR'
        elif emotion == Emotion.JOY:
            return 'JOY'
        elif emotion == Emotion.SADNESS:
            return 'SADNESS'
        elif emotion == Emotion.SURPRISE:
            return 'SURPRISE'
        elif emotion == Emotion.NEUTRAL:
            return 'NEUTRAL'
    return 'UNDERSPECIFIED'

def replace_in_file(file, word, word_replacement):
    for i, line in enumerate(open(file)):
        line.replace(':%s' % word, ':%s' % word_replacement)
        line.replace('"%s' % word, '"%s' % word_replacement)

def get_object_id(memory, category):
    cat_mem = memory.get(casefold_text(category, format='triple'), {'brain_ids': [], 'local_ids': [], 'ids': []})
    l = cat_mem['ids'][:]
    id = l[0]
    tail = l[1:]

    cat_mem['ids'] = tail
    memory[casefold_text(cate

    return id, memory

def sigmoid(z, growth_rate=1):
    return 1 / (1 + np.exp(-z * growth_rate))


    with open(os.path.join(os.path.dirname(__file__), "../queries/{}.rq".format(query_filename))) as fr:
        query = fr.read()
    return query

import os
import re
import numpy as np
from datetime import date
import string

from pepper.brain.utils.constants import CAPITALIZED_TYPES
from pepper.language.utils.atoms import Emotion

def read_query(query_filename):
    """
    Read a query from file and return as a string
    Parameters
    ----------
    query_filename: str name of the query. It will be looked for in the queries folder of this project

    Returns
    -------
    query: str the query with placeholders for the query parameters, as a string to be formatted

    """


# All Rights Reserved.

import sys

from oslo_log import log as logging

from glanceclient import exc
from tacker.common import clients
from tacker.extensions import vnflcm

LOG = logging.getLogger(__name__)

class GlanceClient(object):

    def __init__(self, vim_connection_info, version=None):
        super(GlanceClient, self).__init__()
        self.connection = clients.OpenstackSdkConnection(
            vim_connection_info, version).connection

try:
            return self.connection.image.create_image(
                name, allow_duplicates=True, **fields)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def delete(self, image_id):
        try:
            self.connection.image.delete_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found "
                        "at cleanup", {'image': image_id})

    def import_image(self, image, web_path):
        try:
            self.connection.image.import_image(
                image, method='web-download', uri=web_path)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def get(self, image_id):
        try:
            return self.connection.image.get_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found ",
                        {'image': image_id})


super(GlanceClient, self).__init__()
        self.connection = clients.OpenstackSdkConnection(
            vim_connection_info, version).connection

# All Rights Reserved.

import sys

from oslo_log import log as logging

from glanceclient import exc
from tacker.common import clients
from tacker.extensions import vnflcm

LOG = logging.getLogger(__name__)

class GlanceClient(object):

    def __init__(self, vim_connection_info, version=None):

    def create(self, name, **fields):
        try:
            return self.connection.image.create_image(
                name, allow_duplicates=True, **fields)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def delete(self, image_id):
        try:
            self.connection.image.delete_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found "
                        "at cleanup", {'image': image_id})

    def import_image(self, image, web_path):
        try:
            self.connection.image.import_image(
                image, method='web-download', uri=web_path)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def get(self, image_id):
        try:
            return self.connection.image.get_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found ",
                        {'image': image_id})


# All Rights Reserved.

import sys

from oslo_log import log as logging

from glanceclient import exc
from tacker.common import clients
from tacker.extensions import vnflcm

LOG = logging.getLogger(__name__)

class GlanceClient(object):

    def __init__(self, vim_connection_info, version=None):
        super(GlanceClient

        self.connection = clients.OpenstackSdkConnection(
            vim_connection_info, version).connection

    def create(self, name, **fields):
        try:
            return self.connection.image.create_image(
                name, allow_duplicates=True, **fields)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def delete(self, image_id):
        try:
            self.connection.image.delete_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found "
                        "at cleanup", {'image': image_id})

    def import_image(self, image, web_path):
        try:
            self.connection.image.import_image(
                image, method='web-download', uri=web_path)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def get(self, image_id):
        try:
            return self.connection.image.get_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found ",
                        {'image': image_id})


try:
            self.connection.image.import_image(
                image, method='web-download', uri=web_path)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

# All Rights Reserved.

import sys

from oslo_log import log as logging

from glanceclient import exc
from tacker.common import clients
from tacker.extensions import vnflcm

LOG = logging.getLogger(__name__)

class GlanceClient(object):

    def __init__(self, vim_connection_info, version=None):
        super(GlanceClient, self).__init__()
        self.connection = clients.OpenstackSdkConnection(
            vim_connection_info, version).connection

    def create(self, name, **fields):
        try:
            return self.connection.image.create_image(
                name, allow_duplicates=True, **fields)
        except exc.HTTPException:
            type_, value, tb = sys.exc_info()
            raise vnflcm.GlanceClientException(msg=value)

    def delete(self, image_id):
        try:
            self.connection.image.delete_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found "
                        "at cleanup", {'image': image_id})

    def import_image(self, image, web_path):

    def get(self, image_id):
        try:
            return self.connection.image.get_image(image_id)
        except exc.HTTPNotFound:
            LOG.warning("Image %(image)s created not found ",
                        {'image': image_id})


try:
    from src.gimpy import Graph, DIRECTED_GRAPH
except ImportError:
    from coinor.gimpy import Graph, DIRECTED_GRAPH

'''
Small example for demonstrating flow agumenting path (ford-fulkerson)
algorithm.
This example is taken from youtube lecture series on Advanced Operations
Research by Prof. G.Srinivasan.
Link to the video is:
http://example.com

black: there is no flow on the arc
red  : the flow equals to the arc capacity
green: there is positive flow on the arc, less then capacity.
'''
from __future__ import print_function

if __name__=='__main__':
    g = Graph(type = DIRECTED_GRAPH, display = 'off')
    g.add_node(1,pos='"0,2!"', demand=4, label='(1, 4)')

    g.add_node(3,pos='"2,4!"', demand=0)
    g.add_node(2,pos='"2,0!"', demand=0)
    g.add_node(4,pos='"4,2!"', demand=0)
    g.add_node(6,pos='"6,4!"', demand=0)

    g.add_node(5,pos='"6,0!"', demand=0)
    g.add_node(7,pos='"8,2!"', demand=-4, label='(1, -4)')
    g.add_edge(1, 3, cost=0, capacity=2, label='(0, 2)')
    g.add_edge(1, 2, cost=0, capacity=2, label='(0, 2)')
    g.add_edge(3, 4, cost=1, capacity=1, label='(1, 1)')
    g.add_edge(3, 6, cost=2, capacity=2, label='(2, 2)')
    g.add_edge(2, 4, cost=3, capacity=1, label='(3, 1)')
    g.add_edge(2, 5, cost=5, capacity=1, label='(5, 1)')
    g.add_edge(4, 6, cost=1, capacity=3, label='(1, 3)')
    g.add_edge(4, 7, cost=4, capacity=2, label='(4, 2)')
    g.add_edge(4, 5, cost=1, capacity=1, label='(3, 1)')
    g.add_edge(6, 7, cost=0, capacity=2, label='(0, 2)')
    g.add_edge(5, 7, cost=0, capacity=2, label='(0, 2)')
    g.set_display_mode('off')
    g.display()

    g.min_cost_flow(algo='cycle_canceling')

    nl = list(int(n) for n in g.get_node_list())
    nl.sort()
    for n in nl:
        for m in nl:
            if g.check_edge(n, m):
                print(n, m, g.get_edge_attr(n, m, 'flow'))


from django.conf import settings
from django.core.paginator import Page, Paginator

class EnhancedPaginator(Paginator):
    def __init__(self, object_list, per_page, orphans=None, **kwargs):
        if not isinstance(per_page, int) or per_page < 1:
            per_page = settings.PAGINATE_COUNT

        # Set orphans count based on page size
        if orphans is None:
            orphans = 5 if per_page <= 50 else 10

        super().__init__(object_list, per_page, orphans=orphans, **kwargs)

return EnhancedPage(*args, **kwargs)

class EnhancedPage(Page):
    def smart_pages(self):
        # If less or 5 pages just display each page link
        if self.paginator.num_pages <= 5:
            return self.paginator.page_range

        # Show first page, last page, next/previous two pages, and current page
        n = self.number
        pages_wanted = [1, n - 2, n - 1, n, n + 1, n + 2, self.paginator.num_pages]
        pages_list = sorted(set(self.paginator.page_range).intersection(pages_wanted))

        # Skip markers
        skip_pages = [
            x[1] for x in zip(pages_list[:-1], pages_list[1:]) if (x[1] - x[0] != 1)
        ]
        for i in skip_pages:
            pages_list.insert(pages_list.index(i), False)

        return pages_list

def get_paginate_count(request):
    """
    Determines the number of items to display in one page, based on the following in
    order:
      1. `per_page` URL query parameter (saving it as user preference in the process)
      2. user preference
      3. `PAGINATE_COUNT` setting
    """
    if not request.user.is_authenticated:
        return (
            int(request.GET.get("per_page"))
            if "per_page" in request.GET
            else settings.PAGINATE_COUNT
        )

    if "per_page" in request.GET:
        try:
            per_page = int(request.GET.get("per_page"))
            request.user.preferences.set("pagination.per_page", per_page, commit=True)
            return per_page
        except ValueError:
            pass

    return request.user.preferences.get("pagination.per_page", settings.PAGINATE_COUNT)


from django.conf import settings
from django.core.paginator import Page, Paginator

class EnhancedPaginator(Paginator):
    def __init__(self, object_list, per_page, orphans=None, **kwargs):
        if not isinstance(per_page, int) or per_page < 1:
            per_page = settings.PAGINATE_COUNT

        # Set orphans count based on page size
        if orphans is None:
            orphans = 5 if per_page <= 50 else 10

        super().__init__(object_list, per_page, orphans=orphans, **kwargs)

    def _get_page(self, *args, **kwargs):

class EnhancedPage(Page):
    def smart_pages(self):
        # If less or 5 pages just display each page link
        if self.paginator.num_pages <= 5:
            return self.paginator.page_range

        # Show first page, last page, next/previous two pages, and current page
        n = self.number
        pages_wanted = [1, n - 2, n - 1, n, n + 1, n + 2, self.paginator.num_pages]
        pages_list = sorted(set(self.paginator.page_range).intersection(pages_wanted))

        # Skip markers
        skip_pages = [
            x[1] for x in zip(pages_list[:-1], pages_list[1:]) if (x[1] - x[0] != 1)
        ]
        for i in skip_pages:
            pages_list.insert(pages_list.index(i), False)

        return pages_list

def get_paginate_count(request):
    """
    Determines the number of items to display in one page, based on the following in
    order:
      1. `per_page` URL query parameter (saving it as user preference in the process)
      2. user preference
      3. `PAGINATE_COUNT` setting
    """
    if not request.user.is_authenticated:
        return (
            int(request.GET.get("per_page"))
            if "per_page" in request.GET
            else settings.PAGINATE_COUNT
        )

    if "per_page" in request.GET:
        try:
            per_page = int(request.GET.get("per_page"))
            request.user.preferences.set("pagination.per_page", per_page, commit=True)
            return per_page
        except ValueError:
            pass

    return request.user.preferences.get("pagination.per_page", settings.PAGINATE_COUNT)


from django.conf import settings
from django.core.paginator import Page, Paginator

class EnhancedPaginator(Paginator):
    def __init__(self, object_list, per_page, orphans=None, **kwargs):
        if not isinstance(per_page, int) or per_page < 1:
            per_page = settings.PAGINATE_COUNT

        # Set orphans count based on page size
        if orphans is None:
            orphans = 5 if per_page <= 50 else 10

        super().__init__(object_list, per_page, orphans=orphans, **kwargs)

    def _get_page(self, *args, **kwargs):
        return EnhancedPage(*args, **kwargs)

class EnhancedPage(Page):
    def smart_pages(self):
        # If less or 5 pages just display each page link
        if self.paginator.num_pages <= 5:
            return self.paginator.page_range

        # Show first page, last page, next/previous two pages, and current page
        n = self.nu

        pages_wanted = [1, n - 2, n - 1, n, n + 1, n + 2, self.paginator.num_pages]
        pages_list = sorted(set(self.paginator.page_range).intersection(pages_wanted))

        # Skip markers
        skip_pages = [
            x[1] for x in zip(pages_list[:-1], pages_list[1:]) if (x[1] - x[0] != 1)
        ]
        for i in skip_pages:
            pages_list.insert(pages_list.index(i), False)

        return pages_list

def get_paginate_count(request):
    """
    Determines the number of items to display in one page, based on the following in
    order:
      1. `per_page` URL query parameter (saving it as user preference in the process)
      2. user preference
      3. `PAGINATE_COUNT` setting
    """
    if not request.user.is_authenticated:
        return (
            int(request.GET.get("per_page"))
            if "per_page" in request.GET
            else settings.PAGINATE_COUNT
        )

    if "per_page" in request.GET:
        try:
            per_page = int(request.GET.get("per_page"))
            request.user.preferences.set("pagination.per_page", per_page, commit=True)
            return per_page
        except ValueError:
            pass

    return request.user.preferences.get("pagination.per_page", settings.PAGINATE_COUNT)


try:
            per_page = int(request.GET.get("per_page"))
            request.user.preferences.set("pagination.per_page", per_page, commit=True)
            return per_page
        except ValueError:
            pass

from django.conf import settings
from django.core.paginator import Page, Paginator

class EnhancedPaginator(Paginator):
    def __init__(self, object_list, per_page, orphans=None, **kwargs):
        if not isinstance(per_page, int) or per_page < 1:
            per_page = settings.PAGINATE_COUNT

        # Set orphans count based on page size
        if orphans is None:
            orphans = 5 if per_page <= 50 else 10

        super().__init__(object_list, per_page, orphans=orphans, **kwargs)

    def _get_page(self, *args, **kwargs):
        return EnhancedPage(*args, **kwargs)

class EnhancedPage(Page):
    def smart_pages(self):
        # If less or 5 pages just display each page link
        if self.paginator.num_pages <= 5:
            return self.paginator.page_range

        # Show first page, last page, next/previous two pages, and current page
        n = self.number
        pages_wanted = [1, n - 2, n - 1, n, n + 1, n + 2, self.paginator.num_pages]
        pages_list = sorted(set(self.paginator.page_range).intersection(pages_wanted))

        # Skip markers
        skip_pages = [
            x[1] for x in zip(pages_list[:-1], pages_list[1:]) if (x[1] - x[0] != 1)
        ]
        for i in skip_pages:
            pages_list.insert(pages_list.index(i), False)

        return pages_list

def get_paginate_count(request):
    """
    Determines the number of items to display in one page, based on the following in
    order:
      1. `per_page` URL query parameter (saving it as user preference in the process)
      2. user preference
      3. `PAGINATE_COUNT` setting
    """
    if not request.user.is_authenticated:
        return (
            int(request.GET.get("per_page"))
            if "per_page" in request.GET
            else settings.PAGINATE_COUNT
        )

    if "per_page" in request.GET:


async def __aexit__(
        self: AsyncWebsocketClient,
        _exc_type: Type[BaseException],
        _exc_val: BaseException,
        _trace: TracebackType,
    ) -> None:

"""A client for interacting with the rippled WebSocket API."""
from __future__ import annotations

from collections.abc import AsyncIterator
from types import TracebackType
from typing import Any, Dict, Type

from xrpl.asyncio.clients.async_client import AsyncClient
from xrpl.asyncio.clients.exceptions import XRPLWebsocketException
from xrpl.asyncio.clients.websocket_base import WebsocketBase
from xrpl.models.requests.request import Request

class AsyncWebsocketClient(AsyncClient, WebsocketBase):
    """
    An async client for interacting with the rippled WebSocket API.

    Instead of calling ``open`` and ``close`` yourself, you
    can use a context like so::

        async with AsyncWebsocketClient(url) as client:
            # inside the context the client is open
        # after exiting the context, the client is closed

    Doing this will open and close the client for you and is
    preferred.

    NOTE: if you are not using subscriptions or other WebSocket-only
    features of rippled, you may not need to do anything other than
    open the client and make requests::

        from xrpl.asyncio.clients import AsyncWebsocketClient
        from xrpl.asyncio.ledger import get_fee
        from xrpl.models import Fee

        async with AsyncWebsocketClient(url) as client:
            # using helper functions
            print(await get_fee(client))

            # using raw requests yourself
            print(await client.request(Fee())

    However, if you are using some functionality that makes use of
    subscriptions or other "websocket-y" things, you can iterate over
    the client like so to read incoming messages::

        async with AsyncWebsocketClient(url) as client:
            # inside the context the client is open
            async for message in client:
                # do something with a message
        # after exiting the context, the client is closed

    The recommended way to use this client is to set up a Task
    using the ``asyncio`` library to listen to incoming
    messages and do something with them, but the above will
    work fine if you want to listen indefinitely. This is how
    you can use a Task to listen to messages without blocking::

        import asyncio

        from xrpl.asyncio.clients import AsyncWebsocketClient
        from xrpl.models import Subscribe, Unsubscribe, StreamParameter

        URL = "wss://some-url-to-connect-to"

        async def listener(client):
            async for message in client:
                # do something with a message

        async def main():
            async with AsyncWebsocketClient(URL) as client:
                # set up the `listener` function as a Task
                # so that it doesn't wait for a response, but
                # will "awaken" whenever the `asyncio` event
                # loop toggles to it.
                asyncio.create_task(listener(client))

                # now, the `listener` function will run as if
                # it were "in the background", doing whatever you
                # want as soon as it has a message.

                # now let's subscribe to something. in this case,
                # we can just use `send` instead of `request`
                # because we don't really care about the response
                # since the listener will also get it.
                await client.send(Subscribe(
                    streams=[StreamParameter.LEDGER],
                ))

                # in the meantime, you can continue to do whatever
                # you want and the python `asyncio` event loop
                # will toggle between your code and the listener
                # as messages are ready. let's just sleep. note,
                # you need to use `asyncio.sleep` within
                # async code instead of `time.sleep`, otherwise
                # you will block all the waiting tasks instead of
                # just this code path.
                await asyncio.sleep(5)

                # now that we're done, we can unsubscribe if
                # we like
                await client.send(Unsubscribe(
                    streams=[StreamParameter.LEDGER],
                ))
            # now, outside of the context, the client is closed.
            # the listener task will now never receive a new message.

        if __name__ == "__main__":
            # remember to run your entire program within a
            # `asyncio.run` call.
            asyncio.run(main())
    """

    async def open(self: AsyncWebsocketClient) -> None:
        """Connects the client to the Web Socket API at the given URL."""
        await self._do_open()

    async def close(self: AsyncWebsocketClient) -> None:
        """Closes the connection."""
        await self._do_close()

    async def __aenter__(self: AsyncWebsocketClient) -> AsyncWebsocketClient:
        """
        Enters an async context after opening itself.

        Returns:
            The opened client.
        """
        await self.open()
        return self

"""Exits an async context after closing itself."""
        await self.close()

    async def __aiter__(self: AsyncWebsocketClient) -> AsyncIterator[Dict[str, Any]]:
        """Iterate on received messages."""
        while self.is_open():
            yield await self._do_pop_message()

    async def send(self: AsyncWebsocketClient, request: Request) -> None:
        """
        Submit the request represented by the request to the
        rippled node specified by this client's URL. Unlike ``request``,
        ``send`` does not wait for this request's response. In many cases
        it may be more convenient to use ``request``.

        Arguments:
            request: A Request object representing information about a rippled request.

        Raises:
            XRPLWebsocketException: If there is already an open request by the
                request's ID, or if this WebsocketBase is not open.
        """
        if not self.is_open():
            raise XRPLWebsocketException("Websocket is not open")
        await self._do_send(request)