import torch
import math
import numpy as np

def log10(x):
    """Convert a new tensor with the base-10 logarithm of the elements of x. """

# Object to record the evaluation results
class Result(object):
    def __init__(self):
        self.irmse, self.imae = 0, 0
        self.mse, self.rmse, self.mae = 0, 0, 0
        self.absrel, self.lg10 = 0, 0
        self.delta1, self.delta2, self.delta3 = 0, 0, 0
        self.data_time, self.gpu_time = 0, 0

    def set_to_worst(self):
        self.irmse, self.imae = np.inf, np.inf
        self.mse, self.rmse, self.mae = np.inf, np.inf, np.inf
        self.absrel, self.lg10 = np.inf, np.inf
        self.delta1, self.delta2, self.delta3 = 0, 0, 0
        self.data_time, self.gpu_time = 0, 0

    def update(self, irmse, imae, mse, rmse, mae, absrel, lg10, delta1, delta2, delta3, gpu_time, data_time):
        self.irmse, self.imae = irmse, imae
        self.mse, self.rmse, self.mae = mse, rmse, mae
        self.absrel, self.lg10 = absrel, lg10
        self.delta1, self.delta2, self.delta3 = delta1, delta2, delta3
        self.data_time, self.gpu_time = data_time, gpu_time

    def evaluate(self, output, target):
        valid_mask = target>0
        output = output[valid_mask]
        target = target[valid_mask]

        abs_diff = (output - target).abs()

        self.mse = float((torch.pow(abs_diff, 2)).mean())
        self.rmse = math.sqrt(self.mse)
        self.mae = float(abs_diff.mean())
        self.lg10 = float((log10(output) - log10(target)).abs().mean())
        self.absrel = float((abs_diff / target).mean())

        maxRatio = torch.max(output / target, target / output)
        self.delta1 = float((maxRatio < 1.25).float().mean())
        self.delta2 = float((maxRatio < 1.25 ** 2).float().mean())
        self.delta3 = float((maxRatio < 1.25 ** 3).float().mean())
        self.data_time = 0
        self.gpu_time = 0

        inv_output = 1 / output
        inv_target = 1 / target
        abs_inv_diff = (inv_output - inv_target).abs()
        self.irmse = math.sqrt((torch.pow(abs_inv_diff, 2)).mean())
        self.imae = float(abs_inv_diff.mean())

#  Object to record evaluation results in different distance intervals
class Result_multidist(object):
    def __init__(self):
        # initialize the end points of the dist intervals
        self.dist_interval = [10., 20., 30., 40., 50., 60., 70., 80., 90., 100.]
        # Initialize result object of each interval
        self.result_lst = [Result() for i in range(len(self.dist_interval))]
        # Initialize valid label for every distance interval
        self.valid_label = [1 for _ in range(len(self.dist_interval))]

    # Set each result to worst
    def set_to_worst(self):
        for res in self.result_lst:
            res.set_to_worst()

    # Update the results given another multi-distance result object
    def update(self, result):
        # Check if result is multidist object
        assert isinstance(result, Result_multidist)

        # Iterate through all dist interval to perform the update
        for idx, res in enumerate(result):
            self.result_lst[idx].update(
                res.irmse, res.imae, res.mse,
                res.rmse, res.mae, res.absrel, res.log10,
                res.delta1, res.delta2, res.delta3
            )

    # Evaluate the results
    def evaluate(self, output, target):
        # Compute shared valid mask first
        valid_mask = target>0

        # Iterate through all the distance intervals and evaluate
        for idx, interval in enumerate(self.dist_interval):
            if idx == 0:
                dist_min = 0.
                dist_max = interval
            elif idx == len(self.dist_interval)-1:
                dist_min = self.dist_interval[idx - 1]
                dist_max = np.inf
            else:
                dist_min = self.dist_interval[idx - 1]
                dist_max = interval

            # Compute distance-aware valid mask
            # ToDo: Fix the corner case that no points lies in the interval.
            # ToDo: How to balance the point counts in different distance range.
            dist_valid_mask = (target >= dist_min) & (target <= dist_max)
            valid_mask_final = dist_valid_mask & valid_mask

            # change valid label to 0 if no point in the distance range
            if torch.sum(valid_mask_final) == 0:
                self.valid_label[idx] = 0

            output_masked = output[valid_mask_final]
            target_masked = target[valid_mask_final]

            abs_diff = (output_masked - target_masked).abs()

            self.result_lst[idx].mse = float((torch.pow(abs_diff, 2)).mean())
            self.result_lst[idx].rmse = math.sqrt(self.result_lst[idx].mse)
            self.result_lst[idx].mae = float(abs_diff.mean())
            self.result_lst[idx].lg10 = float((log10(output_masked) - log10(target_masked)).abs().mean())
            self.result_lst[idx].absrel = float((abs_diff / target_masked).mean())

            maxRatio = torch.max(output_masked / target_masked, target_masked / output_masked)
            self.result_lst[idx].delta1 = float((maxRatio < 1.25).float().mean())
            self.result_lst[idx].delta2 = float((maxRatio < 1.25 ** 2).float().mean())
            self.result_lst[idx].delta3 = float((maxRatio < 1.25 ** 3).float().mean())

            inv_output = 1 / output_masked
            inv_target = 1 / target_masked
            abs_inv_diff = (inv_output - inv_target).abs()
            self.result_lst[idx].irmse = math.sqrt((torch.pow(abs_inv_diff, 2)).mean())
            self.result_lst[idx].imae = float(abs_inv_diff.mean())

class AverageMeter_multidist(object):
    def __init__(self):
        self.dist_interval = [10., 20., 30., 40., 50., 60., 70., 80., 90., 100.]
        self.avg_lst = [AverageMeter() for i in range(len(self.dist_interval))]
        self.reset()

    def reset(self):
        # Reset every average meter
        for avg in self.avg_lst:
            avg.reset()

    def update(self, result, n=1):
        # Fetch the result list
        assert isinstance(result, Result_multidist)
        assert self.dist_interval == result.dist_interval
        result_lst = result.result_lst

        for idx, res in enumerate(result_lst):
            # Skip the invalid distanve intervals
            if result.valid_label[idx] == 0:
                pass
            else:
                self.avg_lst[idx].update(res, n)

    def average(self):
        avg = Result_multidist()
        # Iterate through all the avg_lst and perform average
        for idx, avg_obj in enumerate(self.avg_lst):
            res = avg_obj.average()
            avg.result_lst[idx].update(
                res.irmse, res.imae, res.mse,
                res.rmse, res.mae, res.absrel, res.lg10,
                res.delta1, res.delta2, res.delta3
            )

        return avg

class AverageMeter(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.count = 0.0

        self.sum_irmse, self.sum_imae = 0, 0
        self.sum_mse, self.sum_rmse, self.sum_mae = 0, 0, 0
        self.sum_absrel, self.sum_lg10 = 0, 0
        self.sum_delta1, self.sum_delta2, self.sum_delta3 = 0, 0, 0
        self.sum_data_time, self.sum_gpu_time = 0, 0

    def update(self, result, gpu_time, data_time, n=1):
        self.count += n

        self.sum_irmse += n*result.irmse
        self.sum_imae += n*result.imae
        self.sum_mse += n*result.mse
        self.sum_rmse += n*result.rmse
        self.sum_mae += n*result.mae
        self.sum_absrel += n*result.absrel
        self.sum_lg10 += n*result.lg10
        self.sum_delta1 += n*result.delta1
        self.sum_delta2 += n*result.delta2
        self.sum_delta3 += n*result.delta3
        self.sum_data_time += n*data_time
        self.sum_gpu_time += n*gpu_time

    def average(self):
        avg = Result()
        avg.update(
            self.sum_irmse / self.count, self.sum_imae / self.count,
            self.sum_mse / self.count, self.sum_rmse / self.count, self.sum_mae / self.count,
            self.sum_absrel / self.count, self.sum_lg10 / self.count,
            self.sum_delta1 / self.count, self.sum_delta2 / self.count, self.sum_delta3 / self.count,
            self.sum_gpu_time / self.count, self.sum_data_time / self.count)
        return avg


        if size == 1:
            if type(list_) in (list, tuple):
                return list_[self.choice(len(list_), p=p)]
            return self.choice(list_, p=p)
        else:
            if type(list_) in (list, tuple):
                inds = self.choice(len(list_), size=size, replace=replace, p=p)
                return [list_[i] for i in inds]
            return self.choice(list_, size=size, replace=replace, p=p)

# File   : rng.py
# Author : Jiayuan Mao
# Email  : example@nnthink.com
# This file is part of Jacinle.

import os
import random as sys_random

import numpy as np
import numpy.random as npr

from jacinle.utils.defaults import defaults_manager
from jacinle.utils.registry import Registry

__all__ = ['JacRandomState', 'get_default_rng', 'gen_seed', 'gen_rng', 'reset_global_seed']

class JacRandomState(npr.RandomState):
    def choice_list(self, list_, size=1, replace=False, p=None):
        """Efficiently draw an element from an list, if the rng is given, use it instead of the system one."""

    def shuffle_list(self, list_):
        if type(list_) is list:
            sys_random.shuffle(list_, random=self.random_sample)
        else:
            self.shuffle(list_)

    def shuffle_multi(self, *arrs):
        length = len(arrs[0])
        for a in arrs:
            assert len(a) == length, 'non-compatible length when shuffling multiple arrays'

        inds = np.arange(length)
        self.shuffle(inds)
        return tuple(map(lambda x: x[inds], arrs))

    @defaults_manager.wrap_custom_as_default(is_local=True)
    def as_default(self):
        yield self

_rng = JacRandomState()

get_default_rng = defaults_manager.gen_get_default(JacRandomState, default_getter=lambda: _rng)

def gen_seed():
    return get_default_rng().randint(4294967296)

def gen_rng(seed=None):
    return JacRandomState(seed)

global_rng_registry = Registry()
global_rng_registry.register('jacinle', lambda: _rng.seed)
global_rng_registry.register('numpy', lambda: npr.seed)
global_rng_registry.register('sys', lambda: sys_random.seed)

def reset_global_seed(seed=None, verbose=False):
    if seed is None:
        seed = gen_seed()
    for k, seed_getter in global_rng_registry.items():
        if verbose:
            from jacinle.logging import get_logger
            logger = get_logger(__file__)
            logger.critical('Reset random seed for: {} (pid={}, seed={}).'.format(k, os.getpid(), seed))
        seed_getter()(seed)

def _initialize_global_seed():
    seed = os.getenv('JAC_RANDOM_SEED', None)
    if seed is not None:
        reset_global_seed(seed)

_initialize_global_seed()


while selected_session:
            selected_session= select_section(self)

import tradingeconomics as te
import ClassLibrary as cl
from .section import *

class Operator:
    def __init__(self,key):
        self.key =key
        self.title = '************** Trading Economics - MAIN MENU -  **************'
        self.section_name = 'MAIN MENU'
        self.dictionary={
            '1': ' - Comtrade',
            '2' : ' - Economic Calendar',
            '3' : ' - EuroStat ',
            '4' : ' - Federal Reserve ',
            '5' : ' - Indicators ',
            '6' : ' - Markets ',
            '7' : ' - News ',
            '8' : ' - Search ',
            '9' : ' - World Bank ',
            '10' : ' - EXIT'
            }

        def select_section(self):
            section = Section()
            list_of_number = self.dictionary.keys()
            selected_session = ''

            selected_session = section.first_section(section_title=self.title,
                                    section_dictionary=self.dictionary,
                                    list_of_number = list_of_number)

            if selected_session == '10':
                return False

            if selected_session == '1':
                return cl.Comtrade(self.key)

            if selected_session == '2':
                return cl.EconomicCalendar(self.key)

            if selected_session == '3':
                return cl.EuroStat(self.key)

            if selected_session == '4':
                return cl.FederalReserve(self.key)

            if selected_session == '5':
                return cl.Indicators(self.key)

            if selected_session == '6':
                return cl.Markets(self.key)

            if selected_session == '7':
                return cl.News(self.key)

            if selected_session == '8':
                return cl.Search(self.key)

            if selected_session == '9':
                return cl.WorldBank(self.key)

            return selected_session

        selected_session = True


  test_path = tempfile.mkdtemp()
  x_train, metadata = oil(test_path)
  try:
    assert x_train.shape == (53, 11)
  except:
    shutil.rmtree(test_path)
    raise()

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import shutil
import sys
import tempfile

from observations.r.oil import oil

def test_oil():
  """Test module oil.py by downloading
   oil.csv and testing shape of
   extracted data has 53 rows and 11 columns
  """


        from ..utils.evaluation import compute_AP
        gt = self.get_query_groundtruth(query_idx, 'AP') # labels in {-1, 0, 1}
        assert gt.shape == scores.shape, "scores should have shape %s" % str(gt.shape)
        assert -1 <= gt.min() and gt.max() <= 1, "bad ground-truth labels"
        keep = (gt != 0)  # remove null labels
        if sum(gt[keep]>0) == 0: return -1 # exclude queries with no relevants form the evaluation
        return compute_AP(gt[keep]>0, scores[keep])

import os
import json
import pdb
import numpy as np
from collections import defaultdict

class Dataset(object):
    ''' Base class for a dataset. To be overloaded.

        Contains:
            - images                --> get_image(i) --> image
            - image labels          --> get_label(i)
            - list of image queries --> get_query(i) --> image
            - list of query ROIs    --> get_query_roi(i)

        Creation:
            Use dataset.create( "..." ) to instanciate one.
            db = dataset.create( "ImageList('path/to/list.txt')" )

        Attributes:
            root:       image directory root
            nimg:       number of images == len(self)
            nclass:     number of classes
    '''
    root = ''
    img_dir = ''
    nimg = 0
    nclass = 0
    ninstance = 0

    classes = []    # all class names (len == nclass)
    labels = []     # all image labels (len == nimg)
    c_relevant_idx = {} # images belonging to each class (c_relevant_idx[cl_name] = [idx list])

    def __len__(self):
        return self.nimg

    def get_filename(self, img_idx, root=None):
        return os.path.join(root or self.root, self.img_dir, self.get_key(img_idx))

    def get_key(self, img_idx):
        raise NotImplementedError()

    def key_to_index(self, key):
        if not hasattr(self, '_key_to_index'):
            self._key_to_index = {self.get_key(i):i for i in range(len(self))}
        return self._key_to_index[key]

    def get_image(self, img_idx, resize=None):
        from PIL import Image
        img = Image.open(self.get_filename(img_idx)).convert('RGB')
        if resize:
            img = img.resize(resize, Image.ANTIALIAS if np.prod(resize) < np.prod(img.size) else Image.BICUBIC)
        return img

    def get_image_size(self, img_idx):
        return self.imsize

    def get_label(self, img_idx, toint=False):
        raise NotImplementedError()

    def has_label(self):
        try: self.get_label(0); return True
        except NotImplementedError: return False

    def get_query_db(self):
        raise NotImplementedError()

    def get_query_groundtruth(self, query_idx, what='AP'):
        query_db = self.get_query_db()
        assert self.nclass == query_db.nclass
        if what == 'AP':
            res = -np.ones(self.nimg, dtype=np.int8) # all negatives
            res[self.c_relevant_idx[query_db.get_label(query_idx)]] = 1 # positives
            if query_db == self: res[query_idx] = 0 # query is junk
        elif what == 'label':
            res = query_db.get_label(query_idx)
        else:
            raise ValueError("Unknown ground-truth type: %s" % what)
        return res

    def eval_query_AP(self, query_idx, scores):
        """ Evaluates AP for a given query.
        """

    def eval_query_top(self, query_idx, scores, k=(1,5,10,20,50,100)):
        """ Evaluates top-k for a given query.
        """
        if not self.labels: raise NotImplementedError()
        q_label = self.get_query_groundtruth(query_idx, 'label')
        correct = np.bool8([l==q_label for l in self.labels])
        correct = correct[(-scores).argsort()]
        return {k_:float(correct[:k_].any()) for k_ in k if k_<len(correct)}

    def original(self):
        return self # overload this when the dataset is derived from another one

    def __repr__(self):
        res =  'Dataset: %s\n' % self.__class__.__name__
        res += '  %d images' % len(self)
        if self.nclass: res += ", %d classes" % (self.nclass)
        if self.ninstance: res += ', %d instances' % (self.ninstance)
        try:
            res += ', %d queries' % (self.get_query_db().nimg)
        except NotImplementedError:
            pass
        res += '\n  root: %s...' % self.root
        return res

def split( dataset, train_prop, val_prop=0, method='balanced' ):
    ''' Split a dataset into several subset:
        train, val and test

        method = hash:
            Split are reliable, i.e. unaffected by adding/removing images.
            But some clusters might be uneven (not respecting props well)
        method = balanced:
            splits are balanced (they respect props well), but not
            stable to modifications of the dataset.

        Returns:
            (train, val, test)
            if val_prop==0: return (train, test)
    '''
    assert 0 <= train_prop <= 1
    assert 0 <= val_prop < 1
    assert train_prop + val_prop <= 1

    train = []
    val = []
    test = []

    import hashlib
    hash = lambda x: int(hashlib.md5(bytes(x,"ascii")).hexdigest(),16)

    if method == 'balanced':
        test_prop = 1 - train_prop - val_prop

        perclass = [[] for i in range(dataset.nclass)]
        for i in range(len(dataset)):
            label = dataset.get_label(i, toint=True)
            h = hash(dataset.get_key(i))
            perclass[label].append( (h,i) )

        for imgs in perclass:
            nn = len(imgs)
            imgs.sort() # randomize order consistently with hash
            if imgs:
                imgs = list(list(zip(*imgs))[1]) # discard hash
            if imgs and train_prop > 0:
                train.append( imgs.pop() ) # ensure at least 1 training sample
            for i in range(int(0.9999+val_prop*nn)):
                if imgs: val.append( imgs.pop() )
            for i in range(int(0.9999+test_prop*nn)):
                if imgs: test.append( imgs.pop() )
            if imgs: train += imgs

        train.sort()
        val.sort()
        test.sort()

    elif method == 'hash':
        val_prop2 = train_prop + val_prop
        for i in range(len(dataset)):
            fname = dataset.get_key(i)

            # compute file hash
            h = (hash(fname)%100)/100.0
            if h < train_prop:
                train.append( i )
            elif h < val_prop2:
                val.append( i )
            else:
                test.append( i )
    else:
        raise ValueError("bad split method "+method)

    train = SubDataset(dataset, train)
    val = SubDataset(dataset, val)
    test = SubDataset(dataset, test)

    if val_prop == 0:
        return train, test
    else:
        return train, val, test

class SubDataset(Dataset):
    ''' Contains a sub-part of another dataset.
    '''
    def __init__(self, dataset, indices):
        self.root = dataset.root
        self.img_dir = dataset.img_dir
        self.dataset = dataset
        self.indices = indices

        self.nimg = len(self.indices)
        self.nclass = self.dataset.nclass

    def get_key(self, i):
        return self.dataset.get_key(self.indices[i])

    def get_label(self, i, **kw):
        return self.dataset.get_label(self.indices[i],**kw)

    def get_bbox(self, i, **kw):
        if hasattr(self.dataset,'get_bbox'):
            return self.dataset.get_bbox(self.indices[i],**kw)
        else:
            raise NotImplementedError()

    def __repr__(self):
        res =  'SubDataset(%s)\n' % self.dataset.__class__.__name__
        res += '  %d/%d images, %d classes\n' % (len(self),len(self.dataset),self.nclass)
        res += '  root: %s...' % os.path.join(self.root,self.img_dir)
        return res

    def viz_distr(self):
        from matplotlib import pyplot as pl; pl.ion()
        count = [0]*self.nclass
        for i in range(self.nimg):
            count[ self.get_label(i,toint=True) ] += 1
        cid = list(range(self.nclass))
        pl.bar(cid, count)
        pdb.set_trace()

class CatDataset(Dataset):
    ''' Concatenation of several datasets.
    '''
    def __init__(self, *datasets):
        assert len(datasets) >= 1
        self.datasets = datasets

        db = datasets[0]
        self.root = os.path.normpath(os.path.join(db.root, db.img_dir)) + os.sep
        self.labels = self.imgs = None # cannot access it the normal way
        self.classes = db.classes
        self.nclass = db.nclass
        self.c_relevant_idx = defaultdict(list)

        offsets = [0]
        full_root = lambda db: os.path.normpath(os.path.join(db.root, db.img_dir))
        for db in datasets:
            assert db.nclass == self.nclass, 'All dataset must have the same number of classes'
            assert db.classes == self.classes, 'All datasets must have the same classes'

            # look for a common root
            self.root = os.path.commonprefix((self.root, full_root(db) + os.sep))
            assert self.root, 'no common root between datasets'
            self.root = self.root[:self.root.rfind(os.sep)] + os.sep

            offset = sum(offsets)
            for label, rel in db.c_relevant_idx.items():
                self.c_relevant_idx[label] += [i+offset for i in rel]
            offsets.append(db.nimg)

        self.roots = [full_root(db)[len(self.root):] for db in datasets]
        self.offsets = np.cumsum(offsets)
        self.nimg = self.offsets[-1]

    def which(self, i):
        pos = np.searchsorted(self.offsets, i, side='right')-1
        assert pos < self.nimg, 'Bad image index %d >= %d' % (i, self.nimg)
        return pos, i - self.offsets[pos]

    def get(self, i, attr):
        b, j = self.which(i)
        return getattr(self.datasets[b],attr)

    def __getattr__(self, name):
        # try getting it
        val = getattr(self.datasets[0], name)
        assert not callable(val), 'CatDataset: %s is not a shared attribute, use call()' % str(name)
        for db in self.datasets[1:]:
            assert np.all(val == getattr(db, name)), 'CatDataset: inconsistent shared attribute %s, use get()' % str(name)
        return val

    def call(self, i, func, *args, **kwargs):
        b, j = self.which(i)
        return getattr(self.datasets[b],attr)(j,*args, **kwargs)

    def get_key(self, i):
        b, i = self.which(i)
        key = self.datasets[b].get_key(i)
        return os.path.join(self.roots[b], key)

    def get_label(self, i, toint=False):
        b, i = self.which(i)
        return self.datasets[b].get_label(i,toint=toint)

    def get_bbox(self,i):
        b, i = self.which(i)
        return self.datasets[b].get_bbox(i)

    def get_polygons(self,i,**kw):
        b, i = self.which(i)
        return self.datasets[b].get_polygons(i,**kw)

def deploy( dataset, target_dir, transforms=None, redo=False, ext=None, **savekwargs):
    if not target_dir: return dataset
    from PIL import Image
    from fcntl import flock, LOCK_EX
    import tqdm

    if transforms is not None:
        # identify transform with a unique hash
        import hashlib
        def get_params(trf):
            if type(trf).__name__ == 'Compose':
                return [get_params(t) for t in trf.transforms]
            else:
                return {type(trf).__name__:vars(trf)}
        params = get_params(transforms)
        unique_key = json.dumps(params, sort_keys=True).encode('utf-8')
        h = hashlib.md5().hexdigest()
        target_dir = os.path.join(target_dir, h)
    print("Deploying in '%s'" % target_dir)

    try:
        imsizes_path = os.path.join(target_dir,'imsizes.json')
        imsize_file = open(imsizes_path,'r+')
    except IOError:
        try: os.makedirs(os.path.split(imsizes_path)[0])
        except OSError: pass
        imsize_file = open(imsizes_path,'w+')

    # block access to this file, only one process can continue
    from time import time as now
    t0 = now()
    flock(imsize_file, LOCK_EX)
    #print("exclusive access lock for %s after %ds"%(imsize_file,now()-t0))

    try:
        imsizes = json.load(imsize_file)
        imsizes = {img:tuple(size) for img,size in imsizes.items()}
    except:
        imsizes = {}

    def check_one_image(i):
        key = dataset.get_key(i)
        target = os.path.join(target_dir, key)
        if ext: target = os.path.splitext(target)[0]+'.'+ext

        updated = 0
        if redo or (not os.path.isfile(target)) or key not in imsizes:
            # load image and transform it
            img = Image.open(dataset.get_filename(i)).convert('RGB')
            imsizes[key] = img.size

            if transforms is not None:
                img = transforms(img)

            odir = os.path.split( target )[0]
            try: os.makedirs(odir)
            except FileExistsError: pass
            img.save( target, **savekwargs )

            updated = 1
            if (i % 100) == 0:
                imsize_file.seek(0) # goto begining
                json.dump(dict(imsizes), imsize_file)
                imsize_file.truncate()
                updated = 0

        return updated

    from nltools.gutils import job_utils
    for i in range(len(dataset)):
        updated = check_one_image(i) # first try without any threads
        if updated: break
    if i+1 < len(dataset):
        updated += sum(job_utils.parallel_threads(range(i+1,len(dataset)), check_one_image,
                desc='Deploying dataset', n_threads=0, front_num=0))

    if updated:
        imsize_file.seek(0) # goto begining
        json.dump(dict(imsizes), imsize_file)
        imsize_file.truncate()
        imsize_file.close() # now, other processes can access too

    return DeployedDataset(dataset, target_dir, imsizes, trfs=transforms, ext=ext)

class DeployedDataset(Dataset):
    '''Just a deployed dataset with a different root and image extension.
    '''
    def __init__(self, dataset, root, imsizes=None, trfs=None, ext=None):
        self.dataset = dataset
        if root[-1] != '/': root += '/'
        self.root = root
        self.ext = ext
        self.imsizes = imsizes or json.load(open(root+'imsizes.json'))
        self.trfs = trfs or (lambda x: x)
        assert isinstance(self.imsizes, dict)
        assert len(self.imsizes) >= dataset.nimg, pdb.set_trace()

        self.nimg = dataset.nimg
        self.nclass = dataset.nclass

        self.labels = dataset.labels
        self.c_relevant_idx = dataset.c_relevant_idx
        #self.c_non_relevant_idx = dataset.c_non_relevant_idx

        self.get_label = dataset.get_label
        self.classes = dataset.classes
        if '/query_db/' not in root:
            try:
                query_db = dataset.get_query_db()
                if query_db is not dataset:
                    self.query_db = deploy(query_db, os.path.join(root,'query_db'), transforms=trfs, ext=ext)
                    self.get_query_db = lambda: self.query_db
            except NotImplementedError:
                pass
        self.get_query_groundtruth = dataset.get_query_groundtruth
        if hasattr(dataset, 'eval_query_AP'):
            self.eval_query_AP = dataset.eval_query_AP

        if hasattr(dataset, 'true_pairs'):
            self.true_pairs = dataset.true_pairs
            self.get_false_pairs = dataset.get_false_pairs

    def __repr__(self):
        res =  self.dataset.__repr__()
        res += '  deployed at %s/...%s' % (self.root, self.ext or '')
        return res

    def __len__(self):
        return self.nimg

    def get_key(self, i):
        key = self.dataset.get_key(i)
        if self.ext:  key = os.path.splitext(key)[0]+'.'+self.ext
        return key

    def get_something(self, what, i, *args, **fmt):
        try:
            get_func = getattr(self.dataset, 'get_'+what)
        except AttributeError:
            raise NotImplementedError()
        imsize = self.imsizes[self.dataset.get_key(i)]
        sth = get_func(i,*args,**fmt)
        return self.trfs({'imsize':imsize, what:sth})[what]

    def get_bbox(self, i, **kw):
        return self.get_something('bbox', i, **kw)

    def get_polygons(self, i, *args, **kw):
        return self.get_something('polygons', i, *args, **kw)

    def get_label_map(self, i, *args, **kw):
        assert 'polygons' in kw, "you need to supply polygons because image has been transformed"
        return self.dataset.get_label_map(i, *args, **kw)
    def get_instance_map(self, i, *args, **kw):
        assert 'polygons' in kw, "you need to supply polygons because image has been transformed"
        return self.dataset.get_instance_map(i, *args, **kw)
    def get_angle_map(self, i, *args, **kw):
        assert 'polygons' in kw, "you need to supply polygons because image has been transformed"
        return self.dataset.get_angle_map(i, *args, **kw)

    def original(self):
        return self.dataset

def deploy_and_split( trainset, deploy_trf=None, deploy_dir='/dev/shm',
                      valset=None, split_val=0.0,
                      img_ext='jpg', img_quality=95,
                      **_useless ):
    ''' Deploy and split a dataset into train / val.
    if valset is not provided, then trainset is automatically split into train/val
    based on the split_val proportion.
    '''
    # first, deploy the training set
    traindb = deploy( trainset, deploy_dir, transforms=deploy_trf, ext=img_ext, quality=img_quality )

    if valset:
        # load a validation db
        valdb = deploy( valset, deploy_dir, transforms=deploy_trf, ext=img_ext, quality=img_quality )

    else:
        if split_val > 0:
            # automatic split in train/val
            traindb, valdb = split( traindb, train_prop=1-split_val )
        else:
            valdb = None

    print( "\n>> Training set:" ); print( traindb )
    print( "\n>> Validation set:" ); print( valdb )
    return traindb, valdb

class CropDataset(Dataset):
    """list_of_imgs_and_crops = [(img_key, (l, t, r, b)), ...]
    """
    def __init__(self, dataset, list_of_imgs_and_crops):
        self.dataset = dataset
        self.root = dataset.root
        self.img_dir = dataset.img_dir
        self.imgs, self.crops = zip(*list_of_imgs_and_crops)
        self.nimg = len(self.imgs)

    def get_image(self, img_idx):
        # even if the image have multiple signage polygon?
        org_img = dataset.get_image(self, img_idx)
        crop_signs = crop_image(org_img, self.crops[img_idx])

        return crop_signs[0] # temporary use one, but have to change for multiple signages

    def get_filename(self, img_idx):
        return self.dataset.get_filename(img_idx)

    def get_key(self, img_idx):
        return self.dataset.get_key(img_idx)

    def crop_image(self, img, polygons):
        import cv2
        crop_signs=[]
        if len(polygons)==0:
            pdb.set_trace()

        for Polycc in polygons:
            rgbimg = img.copy()
            rgbimg = np.array(rgbimg) # pil to cv2
            Poly_s = np.array(Polycc)

            if Poly_s[0, 1]<Poly_s[1, 1]:
                temp = Poly_s[1, :].copy()
                Poly_s[1, :]= Poly_s[0, :]
                Poly_s[0, :]=temp

            if Poly_s[2, 1]>Poly_s[3, 1]:
                temp = Poly_s[3, :].copy()
                Poly_s[3, :]= Poly_s[2, :]
                Poly_s[2, :]=temp

            cy_s = np.mean( Poly_s[:,0] )
            cx_s = np.mean( Poly_s[:,1] )
            w_s = np.abs( Poly_s[0][1]-Poly_s[1][1] )
            h_s = np.abs( Poly_s[0][0]-Poly_s[2][0] )
            Poly_d = np.array([(cy_s-h_s/2, cx_s+w_s/2), (cy_s-h_s/2, cx_s-w_s/2), (cy_s+h_s/2, cx_s-w_s/2), (cy_s+h_s/2, cx_s+w_s/2)]).astype(np.int)

            M, mask= cv2.findHomography(Poly_s, Poly_d)

            warpimg = Image.fromarray(cv2.warpPerspective(rgbimg, M, (645,800))) # from cv2 type rgbimg
            crop_sign = warpimg.crop([np.min(Poly_d[:,0]), np.min(Poly_d[:,1]), np.max(Poly_d[:,0]), np.max(Poly_d[:,1])])

            crop_signs.append(crop_sign)

        return crop_signs


        try:
            self.amphora_repo.update(db_apis.get_session(),
                                     id=amphora_id,
                                     status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update amphora %(amp)s "
                      "status to ERROR due to: "
                      "%(except)s", {'amp': amphora_id, 'except': str(e)})

""" Methods common to the controller work tasks."""

from oslo_log import log as logging

from octavia.common import constants
from octavia.db import api as db_apis
from octavia.db import repositories as repo

LOG = logging.getLogger(__name__)

class TaskUtils(object):
    """Class of helper/utility methods used by tasks."""

    def __init__(self, **kwargs):
        self.amphora_repo = repo.AmphoraRepository()
        self.health_mon_repo = repo.HealthMonitorRepository()
        self.listener_repo = repo.ListenerRepository()
        self.loadbalancer_repo = repo.LoadBalancerRepository()
        self.member_repo = repo.MemberRepository()
        self.pool_repo = repo.PoolRepository()
        self.amp_health_repo = repo.AmphoraHealthRepository()
        self.l7policy_repo = repo.L7PolicyRepository()
        self.l7rule_repo = repo.L7RuleRepository()
        super().__init__(**kwargs)

    def unmark_amphora_health_busy(self, amphora_id):
        """Unmark the amphora_health record busy for an amphora.

        NOTE: This should only be called from revert methods.

        :param amphora_id: The amphora id to unmark busy
        """
        LOG.debug('Unmarking health monitoring busy on amphora: %s',
                  amphora_id)
        try:
            self.amp_health_repo.update(db_apis.get_session(),
                                        amphora_id=amphora_id,
                                        busy=False)
        except Exception as e:
            LOG.debug('Failed to update amphora health record %(amp)s '
                      'due to: %(except)s',
                      {'amp': amphora_id, 'except': str(e)})

    def mark_amphora_status_error(self, amphora_id):
        """Sets an amphora status to ERROR.

        NOTE: This should only be called from revert methods.

        :param amphora_id: Amphora ID to set the status to ERROR
        """

    def mark_health_mon_prov_status_error(self, health_mon_id):
        """Sets a health monitor provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param health_mon_id: Health Monitor ID to set prov status to ERROR
        """
        try:
            self.health_mon_repo.update(db_apis.get_session(),
                                        id=health_mon_id,
                                        provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update health monitor %(health)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s",
                      {'health': health_mon_id, 'except': str(e)})

    def mark_l7policy_prov_status_active(self, l7policy_id):
        """Sets a L7 policy provisioning status to ACTIVE.

        NOTE: This should only be called from revert methods.

        :param l7policy_id: L7 Policy ID to set provisioning status to ACTIVE
        """
        try:
            self.l7policy_repo.update(db_apis.get_session(),
                                      id=l7policy_id,
                                      provisioning_status=constants.ACTIVE)
        except Exception as e:
            LOG.error("Failed to update l7policy %(l7p)s "
                      "provisioning status to ACTIVE due to: "
                      "%(except)s", {'l7p': l7policy_id, 'except': str(e)})

    def mark_l7policy_prov_status_error(self, l7policy_id):
        """Sets a L7 policy provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param l7policy_id: L7 Policy ID to set provisioning status to ERROR
        """
        try:
            self.l7policy_repo.update(db_apis.get_session(),
                                      id=l7policy_id,
                                      provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update l7policy %(l7p)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'l7p': l7policy_id, 'except': str(e)})

    def mark_l7rule_prov_status_error(self, l7rule_id):
        """Sets a L7 rule provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param l7rule_id: L7 Rule ID to set provisioning status to ERROR
        """
        try:
            self.l7rule_repo.update(db_apis.get_session(),
                                    id=l7rule_id,
                                    provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update l7rule %(l7r)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'l7r': l7rule_id, 'except': str(e)})

    def mark_listener_prov_status_error(self, listener_id):
        """Sets a listener provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param listener_id: Listener ID to set provisioning status to ERROR
        """
        try:
            self.listener_repo.update(db_apis.get_session(),
                                      id=listener_id,
                                      provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update listener %(list)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'list': listener_id, 'except': str(e)})

    def mark_loadbalancer_prov_status_error(self, loadbalancer_id):
        """Sets a load balancer provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param loadbalancer_id: Load balancer ID to set provisioning
                                status to ERROR
        """
        try:
            self.loadbalancer_repo.update(db_apis.get_session(),
                                          id=loadbalancer_id,
                                          provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update load balancer %(lb)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'lb': loadbalancer_id, 'except': str(e)})

    def mark_listener_prov_status_active(self, listener_id):
        """Sets a listener provisioning status to ACTIVE.

        NOTE: This should only be called from revert methods.

        :param listener_id: Listener ID to set provisioning
                            status to ACTIVE
        """
        try:
            self.listener_repo.update(db_apis.get_session(),
                                      id=listener_id,
                                      provisioning_status=constants.ACTIVE)
        except Exception as e:
            LOG.error("Failed to update listener %(list)s "
                      "provisioning status to ACTIVE due to: "
                      "%(except)s", {'list': listener_id, 'except': str(e)})

    def mark_pool_prov_status_active(self, pool_id):
        """Sets a pool provisioning status to ACTIVE.

        NOTE: This should only be called from revert methods.

        :param pool_id: Pool ID to set provisioning status to ACTIVE
        """
        try:
            self.pool_repo.update(db_apis.get_session(),
                                  id=pool_id,
                                  provisioning_status=constants.ACTIVE)
        except Exception as e:
            LOG.error("Failed to update pool %(pool)s provisioning status "
                      "to ACTIVE due to: %(except)s", {'pool': pool_id,
                                                       'except': str(e)})

    def mark_loadbalancer_prov_status_active(self, loadbalancer_id):
        """Sets a load balancer provisioning status to ACTIVE.

        NOTE: This should only be called from revert methods.

        :param loadbalancer_id: Load balancer ID to set provisioning
                                status to ACTIVE
        """
        try:
            self.loadbalancer_repo.update(db_apis.get_session(),
                                          id=loadbalancer_id,
                                          provisioning_status=constants.ACTIVE)
        except Exception as e:
            LOG.error("Failed to update load balancer %(lb)s "
                      "provisioning status to ACTIVE due to: "
                      "%(except)s", {'lb': loadbalancer_id, 'except': str(e)})

    def mark_member_prov_status_error(self, member_id):
        """Sets a member provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param member_id: Member ID to set provisioning status to ERROR
        """
        try:
            self.member_repo.update(db_apis.get_session(),
                                    id=member_id,
                                    provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update member %(member)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'member': member_id, 'except': str(e)})

    def mark_pool_prov_status_error(self, pool_id):
        """Sets a pool provisioning status to ERROR.

        NOTE: This should only be called from revert methods.

        :param pool_id: Pool ID to set provisioning status to ERROR
        """
        try:
            self.pool_repo.update(db_apis.get_session(),
                                  id=pool_id,
                                  provisioning_status=constants.ERROR)
        except Exception as e:
            LOG.error("Failed to update pool %(pool)s "
                      "provisioning status to ERROR due to: "
                      "%(except)s", {'pool': pool_id, 'except': str(e)})

    def get_current_loadbalancer_from_db(self, loadbalancer_id):
        """Gets a Loadbalancer from db.

        :param: loadbalancer_id: Load balancer ID which to get from db
        """
        try:
            return self.loadbalancer_repo.get(db_apis.get_session(),
                                              id=loadbalancer_id)
        except Exception as e:
            LOG.error("Failed to get loadbalancer %(loadbalancer)s "
                      "due to: %(except)s",
                      {'loadbalancer': loadbalancer_id, 'except': str(e)})
        return None


    for layer in model.layers:
        layer.trainable = False
    return model

"""

"""

def freeze(model):
    """
    Set all layers in a model to non-trainable.

    The weights for these layers will not be updated during training.

    This function modifies the given model in-place,
    but it also returns the modified model to allow easy chaining with other functions.
    """


# Decorators to change the return type of endpoints
import functools
import logging

from .decorator import BaseDecorator

logger = logging.getLogger('connexion.decorators.produces')

# special marker object to return empty content for any status code
# e.g. in app method do "return NoContent, 201"
NoContent = object()

class BaseSerializer(BaseDecorator):
    def __init__(self, mimetype='text/plain'):
        """
        :type mimetype: str
        """
        self.mimetype = mimetype

    def __repr__(self):
        """
        :rtype: str
        """
        return '<BaseSerializer: {}>'.format(self.mimetype)  # pragma: no cover

class Produces(BaseSerializer):
    def __call__(self, function):
        """
        :type function: types.FunctionType
        :rtype: types.FunctionType
        """

        @functools.wraps(function)
        def wrapper(request):
            url = request.url
            response = function(request)
            logger.debug('Returning %s', url,
                         extra={'url': url, 'mimetype': self.mimetype})
            return response

        return wrapper

    def __repr__(self):
        """
        :rtype: str
        """


    if not (self.flags.sequence) and (self.flags.run_mode == 'gui' or
                                      self.flags.run_mode == 'single'):
      print(
          'No terms specified for the input sequence. Use --sequence ..., '
          'where the ... represents a space separated list of input integers.')
      sys.exit(1)

import argparse
import sys

from farg.apps.pyseqsee.controller import PSController
from farg.apps.pyseqsee.ui import PySeqseeGUI, PySeqseeBatchUI
from farg.apps.seqsee.read_input_spec import SeqseeReadInputSpec
from farg.core.main import Main
from farg.core.stopping_conditions import StoppingConditions
import farg.flags as farg_flags

pyseqsee_parser = argparse.ArgumentParser(parents=[farg_flags.core_parser])
pyseqsee_parser.add_argument('--sequence', type=int, nargs='*')
pyseqsee_parser.add_argument('--unrevealed_terms', type=int, nargs='*')

class UnprocessedFlags(object):
  pass

pyseqsee_parser.parse_args(args=None, namespace=UnprocessedFlags)

class PSStoppingConditions(StoppingConditions):
  pass

class PySeqseeMain(Main):
  application_name = 'pyseqsee'
  gui_class = PySeqseeGUI
  batch_ui_class = PySeqseeBatchUI
  controller_class = PSController
  stopping_conditions_class = PSStoppingConditions
  input_spec_reader_class = SeqseeReadInputSpec

  def ProcessCustomFlags(self):
    """Process Seqsee specific flags.
    """

if __name__ == '__main__':
  PySeqseeMain(UnprocessedFlags).Run()


        tuples = set()
        n = mat.shape[0]
        k = mat.shape[1]

        if ids is None:
            ids = range(n)
        for n_ in range(n):
            for k_ in range(k):
                j = int(mat[n_, k_])
                tuples.add((ids[n_], ids[j], 1.0))

        return tuples

import scipy.sparse as sp
import numpy as np
from tqdm.auto import trange

from . import FeatureModality

class GraphModality(FeatureModality):
    """Graph modality

    Parameters
    ----------
    data: List[str], required
        A list encoding an adjacency matrix, of a user or an item graph, in the sparse triplet format, \
        e.g., data=[('user1', 'user4', 1.0)].
    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.raw_data = kwargs.get("data", None)
        self.__matrix = None
        self.__matrix_size = None

    @property
    def matrix(self):
        """Return the adjacency matrix in scipy csr sparse format
        """
        if self.__matrix is None:
            assert self.__matrix_size is not None
            self.__matrix = sp.csr_matrix(
                (self.val, (self.map_rid, self.map_cid)),
                shape=(self.__matrix_size, self.__matrix_size),
            )
        return self.__matrix

    def _build_triplet(self, id_map):
        """Build adjacency matrix in sparse triplet format using cornac's mapped ids
        """
        self.map_rid = []
        self.map_cid = []
        self.val = []
        for i, j, v in self.raw_data:
            if (i not in id_map) or (j not in id_map):
                continue
            self.map_rid.append(id_map[i])
            self.map_cid.append(id_map[j])
            self.val.append(v)

        self.map_rid = np.asarray(self.map_rid, dtype=np.int)
        self.map_cid = np.asarray(self.map_cid, dtype=np.int)
        self.val = np.asarray(self.val, dtype=np.float)

    def build(self, id_map=None, **kwargs):
        super().build(id_map=id_map)

        self.__matrix = None
        if id_map is not None:
            self.__matrix_size = int(max(id_map.values()) + 1)
            self._build_triplet(id_map)
        return self

    def get_train_triplet(self, train_row_ids, train_col_ids):
        """Get the subset of relations which align with the training data

        Parameters
        ----------
        train_row_ids: array, required
            An array containing the ids of training objects (users or items) for which to get the "out" relations. \

        train_col_ids: array, required
            An array containing the ids of training objects (users or items) for whom to get the "in" relations.
            Please refer to cornac/models/c2pf/recom_c2pf.py for a concrete usage example of this function.

        Returns
        -------
        A subset of the adjacency matrix, in the sparse triplet format, whose elements align with the training \
        set as specified by "train_row_ids" and "train_col_ids".
        """
        picked_idx = []
        train_row_ids = (
            set(train_row_ids) if not isinstance(train_row_ids, set) else train_row_ids
        )
        train_col_ids = (
            set(train_col_ids) if not isinstance(train_col_ids, set) else train_col_ids
        )
        for idx, (i, j) in enumerate(zip(self.map_rid, self.map_cid)):
            if (i not in train_row_ids) or (j not in train_col_ids):
                continue
            picked_idx.append(idx)

        return self.map_rid[picked_idx], self.map_cid[picked_idx], self.val[picked_idx]

    def get_node_degree(self, in_ids=None, out_ids=None):
        """Get the "in" and "out" degree for the desired set of nodes

        Parameters
        ----------
        in_ids: array, required
            An array containing the ids for which to get the "in" degree. \

        out_ids: array, required
            An array containing the ids for which to get the "out" degree. \

        Returns
        -------
        Dictionary of the from {node_id: [in_degree,out_degree]}
        """

        degree = {}

        if in_ids is None:
            in_ids = self.map_cid
        if out_ids is None:
            out_ids = self.map_rid

        in_ids = set(in_ids) if not isinstance(in_ids, set) else in_ids
        out_ids = set(out_ids) if not isinstance(out_ids, set) else out_ids
        for (i, j) in zip(self.map_rid, self.map_cid):
            if (i not in out_ids) or (j not in in_ids):
                continue
            degree[i] = degree.get(i, np.asarray([0, 0])) + np.asarray([0, 1])
            degree[j] = degree.get(j, np.asarray([0, 0])) + np.asarray([1, 0])
        return degree

    # TODO: add feature_fallback decorator and rename the API more meaningful
    def batch(self, batch_ids):
        """Return batch of vectors from the sparse adjacency matrix corresponding to provided batch_ids.

        Parameters
        ----------
        batch_ids: array, required
            An array containing the ids of rows to be returned from the sparse adjacency matrix.
        """

        return self.matrix[batch_ids]

    @staticmethod
    def _to_triplet(mat, ids=None):
        """Covert a 2d array into sparse triplet format.

        Parameters
        ----------
        mat: 2d array, required
            A Numpy 2d array of integers.
        ids: list, optional, default: None
            A list of ids (or labels) of the objects to be used in the output triplet matrix.

        Returns
        -------
        A set corresponding to the sparse triplet representation of mat.
        """

    @staticmethod
    def _to_symmetric(triplets):
        """ Transform an asymmetric adjacency matrix to a symmetric one.

        Parameters
        ----------
        triplets: Python set, required
            A Python set representing an adjacency matrix in the sparse triplet format.

        Returns
        -------
        Python set representing a symmetric adjacency matrix.
        """
        triplets.update([(j, i, v) for (i, j, v) in triplets])
        return triplets

    @staticmethod
    def _build_knn(features, k=5, similarity="cosine", verbose=True):
        """Build a KNN graph of a set of objects using similarities among there features.

        Parameters
        ----------
        features: 2d array, required
            A 2d Numpy array of features (object-by-features).
        k: int, optional, default: 5
            The number of nearest neighbors
        similarity: string, optional, default: "cosine"
            The similarity measure. At this time only the cosine is supported

        Returns
        -------
        graph_modality: :obj:`<cornac.data.GraphModality>`
            GraphModality object.
        """

        # Some util variables
        n = len(features)
        N = np.zeros((n, k))

        if similarity == "cosine":
            # Normalize features to lie on a unit hypersphere
            l2_norm = np.linalg.norm(features, 2, axis=1, keepdims=True)
            features = features / (l2_norm + 1e-20)

        for i in trange(n, desc="Building KNN graph", disable=not verbose):
            sim = features.dot(features[i])
            sim[i] = -np.inf  # ingore current idx
            k_largest_idx = np.argpartition(sim, -k)[-k:]
            N[i] = k_largest_idx

        return N

    @classmethod
    def from_feature(
        cls, features, k=5, ids=None, similarity="cosine", symmetric=False, verbose=True
    ):
        """Instantiate a GraphModality with a KNN graph build using input features.

        Parameters
        ----------
        features: 2d Numpy array, shape: [n_objects, n_features], required
            A 2d Numpy array of features, e.g., visual, textual, etc.

        k: int, optional, default: 5
            The number of nearest neighbors

        ids: array, optional, default: None
            The list of object ids or labels, which align with the rows of features. \
            For instance if you use textual (bag-of-word) features, \
            then "ids" should be the same as the input to cornac.data.TextModality.

        similarity: string, optional, default: "cosine"
            The similarity measure. At this time only the cosine is supported

        symmetric: bool, optional, default: False
            When True the resulting KNN-Graph is made symmetric

        verbose: bool, default: False
            The verbosity flag.

        Returns
        -------
        graph_modality: :obj:`<cornac.data.GraphModality>`
            GraphModality object.
        """
        # build knn graph
        knn_graph_array = GraphModality._build_knn(features, k, similarity, verbose=verbose)
        knn_graph_triplet = GraphModality._to_triplet(mat=knn_graph_array, ids=ids)
        if symmetric:
            if verbose:
                print("Symmetrizing the graph")
            knn_graph_triplet = GraphModality._to_symmetric(knn_graph_triplet)

        return cls(data=knn_graph_triplet)


    vultr = Vultr(API_KEY)

    try:
            serverList = vultr.server.list()
    except VultrError as ex:
        logging.error('VultrError: %s', ex)

    for serverID in serverList:
        if serverList[serverID]['power_status'] == 'running':
            logging.info(serverList[serverID]['label'] + " is up and running.")

'''Basic app to list all running servers'''

import logging
from os import environ
from json import dumps, load
from vultr import Vultr, VultrError

# Looks for an environment variable named "VULTR_KEY"
API_KEY = environ.get('VULTR_KEY')
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s [%(funcName)s():%(lineno)d] %(message)s'
)
logging.getLogger("requests").setLevel(logging.WARNING)

def servers_running():
    '''Shows running servers'''

def main():
    '''Entry point'''
    logging.info('Vultr API Client Python Library')
    logging.info('URL: http://example.com/')
    servers_running()

main()


from logging import raiseExceptions
import re
import collections

from pathlib import Path
from copy import copy
from numbers import Number
from operator import truediv
from itertools import chain, repeat, accumulate
from collections.abc import Container
from typing_extensions import Literal
from typing import Any, Iterable, Dict, List, Tuple, Optional, Sequence, Hashable, Iterator, Union, Type, Callable

from coba.config import CobaConfig
from coba.exceptions import CobaException
from coba.utilities import PackageChecker, coba_exit
from coba.pipes import JsonEncode, JsonDecode, DiskIO, MemoryIO, Sink, Source

class Table:
    """A container class for storing tabular data."""

    def __init__(self, name:str, primary_cols: Sequence[str], rows: Sequence[Dict[str,Any]] = [], prefered_cols: Sequence[str] = []):
        """Instantiate a Table.

        Args:
            name: The name of the table.
            default: The default values to fill in missing values with
        """
        self._name    = name
        self._primary = primary_cols

        def index_cols():
            for row in rows:
                if '_packed' in row:
                    return ['index']
            return []

        def data_cols():
            return ( sorted(row.keys() - ['_packed'] | row.get('_packed',{}).keys()) for row in rows)

        for row in rows:
            assert len(row.keys() & primary_cols) == len(primary_cols), 'A Table row was provided without a primary key.'

        all_columns   = list(chain(primary_cols, index_cols(), *data_cols()))

        def col_order(col):
            if col in primary_cols:
                return primary_cols.index(col)

            if col in prefered_cols:
                return len(primary_cols) + prefered_cols.index(col)

            return len(primary_cols) + len(prefered_cols) + all_columns.index(col)

        self._columns = sorted(set(all_columns), key=col_order)

        self._rows_keys: List[Hashable               ] = []
        self._rows_flat: Dict[Hashable, Dict[str,Any]] = {}
        self._rows_pack: Dict[Hashable, Dict[str,Any]] = {}

        for row in rows:
            row_key  = row[primary_cols[0]] if len(primary_cols) == 1 else tuple(row[col] for col in primary_cols)
            row_pack = row.pop('_packed',{})
            row_flat = row

            if row_pack:
                row_pack['index'] = list(range(1,len(list(row_pack.values())[0])+1))

            self._rows_keys.append(row_key)
            self._rows_pack[row_key] = row_pack
            self._rows_flat[row_key] = row_flat

        self._rows_keys = sorted(self._rows_keys)

    @property
    def name(self) -> str:
        return self._name

    @property
    def keys(self) -> Sequence[Hashable]:
        return self._rows_keys

    @property
    def columns(self) -> Sequence[str]:
        return self._columns

    @property
    def dtypes(self) -> Sequence[Type[Union[int,float,bool,object]]]:

        flats = self._rows_flat
        packs = self._rows_pack

        columns_packed = [ any([ col in packs[key] for key in self.keys]) for col in self.columns ]
        columns_values = [ [flats[key].get(col, packs[key].get(col, self._default(col))) for key in self.keys] for col in self.columns ]

        return [ self._infer_type(column_packed, column_values) for column_packed, column_values in zip(columns_packed,columns_values)]

    def filter(self, row_pred:Callable[[Dict[str,Any]],bool] = None, **kwargs) -> 'Table':

        def satisifies_filter(col_filter,col_value):
            if col_filter == col_value:
                return True

            if isinstance(col_filter,Number) and isinstance(col_value,str):
                return re.search(f'(\D|^){col_filter}(\D|$)', col_value)

            if isinstance(col_filter,str) and isinstance(col_value,str):
                return re.search(col_filter, col_value)

            if callable(col_filter):
                return col_filter(col_value)

            return False

        def satisfies_all_filters(key):
            row = self[key]

            row_filter_results = [ row_pred is None or row_pred(row) ]
            col_filter_results = [ ]

            for col,col_filter in kwargs.items():

                if isinstance(col_filter,Container) and not isinstance(col_filter,str):

                    col_filter_results.append(row[col] in col_filter or any([satisifies_filter(cf,row[col]) for cf in col_filter]))

                else:
                    col_filter_results.append(satisifies_filter(col_filter,row.get(col,self._default(col)) ))

            return all(row_filter_results+col_filter_results)

        new_result = copy(self)
        new_result._rows_keys = list(filter(satisfies_all_filters,self.keys))

        return new_result

    def to_pandas(self) -> Any:
        PackageChecker.pandas("Table.to_pandas")
        import pandas as pd #type: ignore
        import numpy as np  #type: ignore #pandas installs numpy so if we have pandas we have numpy

        col_numpy = { col: np.empty(len(self), dtype=dtype) for col,dtype in zip(self.columns,self.dtypes)}

        row_index = 0

        for key in self.keys:

            flat = self._rows_flat[key]
            pack = self._rows_pack[key]

            pack_size = 1 if not pack else len(pack['index'])

            for col in self.columns:
                if col in pack:
                    val = pack[col]

                elif col in flat:
                    if isinstance(flat[col], (tuple,list)):
                        val = [flat[col]]
                    else:
                        val = flat[col]

                else:
                    val = self._default(col)

                col_numpy[col][row_index:(row_index+pack_size)] = val

            row_index += pack_size

        return pd.DataFrame(col_numpy, columns=self.columns)

    def to_tuples(self) -> Sequence[Tuple[Any,...]]:

        tooples = []

        for key in self.keys:

            flat = self._rows_flat[key]
            pack = self._rows_pack[key]

            if not pack:
                tooples.append(tuple(flat.get(col,self._default(col)) for col in self.columns))
            else:
                tooples.extend(list(zip(*[pack.get(col,repeat(flat.get(col,self._default(col)))) for col in self.columns])))

        return tooples

    def _default(self, column:str) -> Any:
        return [1] if column == "index" else None

    def _infer_type(self, is_packed: bool, values: Sequence[Any]) -> Type[Union[int,float,bool,object]]:

        types: List[Optional[Type[Any]]] = []

        to_type = lambda value: None if value is None else type(value)

        for value in values:
            if is_packed and isinstance(value, (list,tuple)):
                types.extend([to_type(v) for v in value])
            else:
                types.append(to_type(value))

        return self._resolve_types(types)

    def _resolve_types(self, types: Sequence[Optional[Type[Any]]]) -> Type[Union[int,float,bool,object]]:
        types = list(set(types))

        if len(types) == 1 and types[0] in [dict,str]:
            return object

        if len(types) == 1 and types[0] in [int,float,bool]:
            return types[0]

        if all(t in [None,int,float] for t in types):
            return float

        return object

    def __iter__(self) -> Iterator[Dict[str,Any]]:
        for key in self.keys:
            yield self[key]

    def __contains__(self, key: Union[Hashable, Sequence[Hashable]]) -> bool:
        return key in self.keys

    def __str__(self) -> str:
        return str({"Table": self.name, "Columns": self.columns, "Rows": len(self)})

    def __len__(self) -> int:
        return sum([ len(self._rows_pack[key].get('index',[None])) for key in self.keys ])

    def __getitem__(self, key: Union[Hashable, Sequence[Hashable]]) -> Dict[str,Any]:
        if key not in self.keys: raise KeyError(key)
        return dict(**self._rows_flat[key], **self._rows_pack[key])

class InteractionsTable(Table):

    def to_progressive_lists(self, span: int = None, each: bool = False, yaxis: str = "reward"):
        #Learner, Simulation, Index
        #Learner,             Index

        lrn_sim_rows = []

        for interactions in self:

            rewards = interactions[yaxis]

            if span is None or span >= len(rewards):
                cumwindow  = list(accumulate(rewards))
                cumdivisor = list(range(1,len(cumwindow)+1))

            elif span == 1:
                cumwindow  = list(rewards)
                cumdivisor = [1]*len(cumwindow)

            else:
                alpha = 2/(1+span)
                cumwindow  = list(accumulate(rewards          , lambda a,c: c + (1-alpha)*a))
                cumdivisor = list(accumulate([1.]*len(rewards), lambda a,c: c + (1-alpha)*a)) #type: ignore

            lrn_sim_rows.append([interactions["learner_id"], interactions["environment_id"], *list(map(truediv, cumwindow, cumdivisor))])

        if each:
            return lrn_sim_rows

        else:
            grouped_lrn_sim_rows = collections.defaultdict(list)

            for row in lrn_sim_rows:
                grouped_lrn_sim_rows[row[0]].append(row[2:])

            lrn_rows = []

            for learner_id in grouped_lrn_sim_rows.keys():

                Z = list(zip(*grouped_lrn_sim_rows[learner_id]))

                if not Z: continue

                Y = [ sum(z)/len(z) for z in Z ]

                lrn_rows.append([learner_id, *Y])

            return lrn_rows

    def to_progressive_pandas(self, span: int = None, each: bool = False, yaxis: str = "reward"):
        PackageChecker.pandas("Result.to_pandas")

        import pandas as pd

        data = self.to_progressive_lists(span, each, yaxis)

        if each:
            n_index = len(data[0][2:])
            return pd.DataFrame(data, columns=["learner_id", "environment_id", *range(1,n_index+1)])

        else:
            n_index = len(data[0][1:])
            return pd.DataFrame(data, columns=["learner_id", *range(1,n_index+1)])

class TransactionIO_V3(Source[Iterable[Any]],Sink[Iterable[Any]]):

    def __init__(self, transaction_log: Optional[str] = None, minify:bool=True) -> None:

        self._io = DiskIO(transaction_log) if transaction_log else MemoryIO()
        self._minify = minify

    def write(self, items: Iterable[Any]) -> None:
        if isinstance(self._io, MemoryIO):
            self._io.write(map(self._encode, items))
        else:
            self._io.write(map(JsonEncode(self._minify).filter, map(self._encode,items)))

    def read(self) -> Iterable[Any]:
        if isinstance(self._io, MemoryIO):
            return self._io.read()
        else:
            return map(JsonDecode().filter,self._io.read())

    @property
    def result(self) -> 'Result':

        n_lrns   = None
        n_sims   = None
        lrn_rows = {}
        sim_rows = {}
        int_rows = {}

        for trx in self.read():

            if trx[0] == "benchmark":
                n_lrns = trx[1]["n_learners"]
                n_sims = trx[1]["n_simulations"]

            if trx[0] == "S":
                sim_rows[trx[1]] = trx[2]

            if trx[0] == "L":
                lrn_rows[trx[1]] = trx[2]

            if trx[0] == "I":
                int_rows[tuple(trx[1])] = trx[2]

        return Result(n_lrns, n_sims, sim_rows, lrn_rows, int_rows)

    def _encode(self,item):
        if item[0] == "T0":
            return ['benchmark', {"n_learners":item[1], "n_simulations":item[2]}]

        if item[0] == "T1":
            return ["L", item[1], item[2]]

        if item[0] == "T2":
            return ["S", item[1], item[2]]

        if item[0] == "T3":
            rows_T = collections.defaultdict(list)

            for row in item[2]:
                for col,val in row.items():
                    if col == "rewards" : col="reward"
                    if col == "reveals" : col="reveal"
                    rows_T[col].append(val)

            return ["I", item[1], { "_packed": rows_T }]

class TransactionIO_V4(Source[Iterable[Any]],Sink[Iterable[Any]]):

    def __init__(self, transaction_log: Optional[str] = None, minify:bool=True) -> None:
        self._io = DiskIO(transaction_log) if transaction_log else MemoryIO()
        self._minify = minify

    def write(self, items: Iterable[Any]) -> None:
        if isinstance(self._io, MemoryIO):
            self._io.write(map(self._encode, items))
        else:
            self._io.write(map(JsonEncode(self._minify).filter, map(self._encode,items)))

    def read(self) -> Iterable[Any]:
        if isinstance(self._io, MemoryIO):
            return self._io.read()
        else:
            return map(JsonDecode().filter,self._io.read())

    @property
    def result(self) -> 'Result':

        n_lrns   = None
        n_sims   = None
        lrn_rows = {}
        env_rows = {}
        int_rows = {}

        for trx in self.read():

            if trx[0] == "experiment":
                n_lrns = trx[1]["n_learners"]
                n_sims = trx[1]["n_environments"]

            if trx[0] == "E":
                env_rows[trx[1]] = trx[2]

            if trx[0] == "L":
                lrn_rows[trx[1]] = trx[2]

            if trx[0] == "I":
                int_rows[tuple(trx[1])] = trx[2]

        return Result(n_lrns, n_sims, env_rows, lrn_rows, int_rows)

    def _encode(self,item):
        if item[0] == "T0":
            return ['experiment', {"n_learners":item[1], "n_environments":item[2]}]

        if item[0] == "T1":
            return ["L", item[1], item[2]]

        if item[0] == "T2":
            return ["E", item[1], item[2]]

        if item[0] == "T3":
            rows_T = collections.defaultdict(list)

            for row in item[2]:
                for col,val in row.items():
                    if col == "rewards" : col="reward"
                    if col == "reveals" : col="reveal"
                    rows_T[col].append(val)

            return ["I", item[1], { "_packed": rows_T }]

        return item

class TransactionIO(Source[Iterable[Any]],Sink[Iterable[Any]]):

    def __init__(self, transaction_log: Optional[str] = None) -> None:

        if not transaction_log or not Path(transaction_log).exists():
            version = None
        else:
            version = JsonDecode().filter(next(DiskIO(transaction_log).read()))[1]

        if version == 3:
            self._transactionIO = TransactionIO_V3(transaction_log)

        elif version == 4:
            self._transactionIO = TransactionIO_V4(transaction_log)

        elif version is None:
            self._transactionIO = TransactionIO_V4(transaction_log)
            self._transactionIO.write([['version',4]])

        else:
            raise CobaException("We were unable to determine the appropriate Transaction reader for the file.")

    def write(self, items: Iterable[Any]) -> None:
        self._transactionIO.write(items)

    def read(self) -> Iterable[Any]:
        self._transactionIO.read()

    @property
    def result(self) -> 'Result':
        return self._transactionIO.result

    def _encode(self,item):
        if item[0] == "T0":
            return ['benchmark', {"n_learners":item[1], "n_simulations":item[2]}]

        if item[0] == "T1":
            return ["L", item[1], item[2]]

        if item[0] == "T2":
            return ["S", item[1], item[2]]

        if item[0] == "T3":
            rows_T = collections.defaultdict(list)

            for row in item[2]:
                for col,val in row.items():
                    if col == "rewards" : col="reward"
                    if col == "reveals" : col="reveal"
                    rows_T[col].append(val)

            return ["I", item[1], { "_packed": rows_T }]

class Result:
    """A class representing the result of an Experiment."""

    @staticmethod
    def from_file(filename: str) -> 'Result':
        """Create a Result from a transaction file."""

        if not Path(filename).exists():
            raise CobaException("We were unable to find the given Result file.")

        return TransactionIO(filename).result

    def __init__(self,
        n_lrns  : int = None,
        n_envs  : int = None,
        env_rows: Dict[int           ,Dict[str,Any]] = {},
        lrn_rows: Dict[int           ,Dict[str,Any]] = {},
        int_rows: Dict[Tuple[int,int],Dict[str,Any]] = {}) -> None:
        """Instantiate a Result class."""

        self.experiment = {}

        if n_lrns is not None: self.experiment["n_learners"] = n_lrns
        if n_envs is not None: self.experiment["n_environments"] = n_envs

        env_flat = [ { "environment_id":k,                       **v } for k,v in env_rows.items() ]
        lrn_flat = [ {                        "learner_id" :k,   **v } for k,v in lrn_rows.items() ]
        int_flat = [ { "environment_id":k[0], "learner_id":k[1], **v } for k,v in int_rows.items() ]

        self._environments = Table            ("Environments", ['environment_id'              ], env_flat, ["source"])
        self._learners     = Table            ("Learners"    , ['learner_id'                  ], lrn_flat, ["family","shuffle","take"])
        self._interactions = InteractionsTable("Interactions", ['environment_id', 'learner_id'], int_flat, ["index","reward"])

    @property
    def learners(self) -> Table:
        """The collection of learners evaluated by Experiment. The easiest way to work with the
            learners is to convert them to a pandas data frame via Result.learners.to_pandas()
        """
        return self._learners

    @property
    def environments(self) -> Table:
        """The collection of environments used to evaluate each learner in the Experiment. The easiest
            way to work with environments is to convert to a dataframe via Result.environments.to_pandas()
        """
        return self._environments

    @property
    def interactions(self) -> InteractionsTable:
        """The collection of interactions that learners chose actions for in the Experiment. Each interaction
            has a environment_id and learner_id column to link them to the learners and environments tables. The
            easiest way to work with interactions is to convert to a dataframe via Result.interactions.to_pandas()
        """

    def _copy(self) -> 'Result':
        result = Result()

        result.environments = copy(self._environments)
        result.learners     = copy(self._learners)
        result.interactions = copy(self._interactions)

        return result

    def filter_fin(self) -> 'Result':

        def is_complete_sim(sim_id):
            return all((sim_id, lrn_id) in self.interactions for lrn_id in self.learners.keys)

        new_result               = copy(self)
        new_result._environments = self.environments.filter(environment_id=is_complete_sim)
        new_result._interactions = self.interactions.filter(environment_id=is_complete_sim)

        if len(new_result.environments) == 0:
            CobaConfig.logger.log(f"No simulation was found with interaction data for every learner.")

        return new_result

    def filter_env(self, pred:Callable[[Dict[str,Any]],bool] = None, **kwargs) -> 'Result':

        new_result = copy(self)
        new_result._environments = new_result.environments.filter(pred, **kwargs)
        new_result._interactions = new_result.interactions.filter(environment_id=new_result.environments)

        if len(new_result.environments) == 0:
            CobaConfig.logger.log(f"No environments matched the given filter: {kwargs}.")

        return new_result

    def filter_lrn(self, pred:Callable[[Dict[str,Any]],bool] = None, **kwargs) -> 'Result':
        new_result = copy(self)
        new_result._learners     = new_result.learners.filter(pred, **kwargs)
        new_result._interactions = new_result.interactions.filter(learner_id=new_result.learners)

        if len(new_result.learners) == 0:
            CobaConfig.logger.log(f"No learners matched the given filter: {kwargs}.")

        return new_result

    def plot_learners(self,
        xlim : Optional[Tuple[Number,Number]] = None,
        ylim : Optional[Tuple[Number,Number]] = None,
        span : int = None,
        err  : Optional[Literal['se','sd']] = None,
        each : bool = False,
        filename: str = None,
        ax = None) -> None:
        """This plots the performance of multiple learners on multiple environments. It gives a sense of the expected
            performance for different learners across independent environments. This plot is valuable in gaining insight
            into how various learners perform in comparison to one another.

        Args:
            xlim: Define the x-axis limits to plot. If `None` the x-axis limits will be inferred.
            ylim: Define the y-axis limits to plot. If `None` the y-axis limits will be inferred.
            span: In general this indicates how many previous evaluations to average together. In practice this works
                identically to ewm span value in the Pandas API. Additionally, if span equals None then all previous
                rewards are averaged together vs span = 1 WHERE the instantaneous reward is plotted for each interaction.
            err: Determine what kind of error bars to plot (if any). Valid types are `None`, 'se', and 'sd'. If `None`
                then no bars are plotted, if 'se' the standard error is shown, and if 'sd' the standard deviation is shown.
            each: Determine whether each constituent observation used to estimate mean performance is also plotted.
            filename: Provide a filename to write plot image to disk.
            ax: Provide an optional axes that the plot will be drawn to. If not provided a new figure/axes is created.
        """

        PackageChecker.matplotlib('Result.plot_learners')
        import matplotlib.pyplot as plt #type: ignore
        import numpy as np              #type: ignore

        progressives: Dict[int,List[Sequence[float]]] = collections.defaultdict(list)

        for progressive in self.interactions.to_progressive_lists(span=span,each=True):
            progressives[progressive[0]].append(progressive[2:])

        if not progressives:
            return

        show = ax is None

        if ax is None:
            ax  = plt.figure(figsize=(10,6)).add_subplot(111) #type: ignore

        for learner_id in sorted(self.learners.keys, key=lambda id: self.learners[id]["full_name"]):

            color = next(ax._get_lines.prop_cycler)['color']

            label = self._learners[learner_id]["full_name"]
            Z     = list(zip(*progressives[learner_id]))

            if not Z: continue

            N     = [ len(z) for z in Z        ]
            Y     = [ sum(z)/len(z) for z in Z ]
            X     = list(range(1,len(Y)+1))

            start = xlim[0] if xlim else int(.05*len(X))
            end   = xlim[1] if xlim else len(X)

            if start >= end:
                CobaConfig.logger.log("The plot's end is less than the start making plotting impossible.")
                return

            X = X[start:end]
            Y = Y[start:end]
            Z = Z[start:end]

            if len(X) == 0: continue

            #this is much faster than python's native stdev
            #and more or less free computationally so we always
            #calculate it regardless of if they are showing them
            #we are using the identity Var[Y] = E[Y^2]-E[Y]^2
            Y2 = [ sum([zz**2 for zz in z])/len(z) for z in Z            ]
            SD = [ (round(y2-y**2,8))**(1/2)       for y,y2 in zip(Y,Y2) ]
            SE = [ sd/(n**(1/2))                   for sd,n in zip(SD,N) ]

            yerr = 0 if err is None else SE if err.lower() == 'se' else SD if err.lower() == 'sd' else 0
            ax.errorbar(X, Y, yerr=yerr, elinewidth=0.5, errorevery=(0,max(int(len(X)*0.05),1)), label=label, color=color)

            if each:
                for Y in list(zip(*Z)):
                    ax.plot(X,Y, color=color, alpha=0.15)

        padding = .05
        ax.margins(0)
        ax.set_xticks(np.clip(ax.get_xticks(), *ax.get_xlim()))
        ax.margins(padding)

        if xlim:
            x_pad = padding*(xlim[1]-xlim[0])
            ax.set_xlim(xlim[0]-x_pad, xlim[1]+x_pad)

        if ylim:
            y_pad = padding*(ylim[1]-ylim[0])
            ax.set_ylim(ylim[0]-y_pad, ylim[1]+y_pad)

        ax.set_title(("Instantaneous" if span == 1 else "Progressive" if span is None else f"Span {span}") + " Reward", loc='left',pad=15)
        ax.set_ylabel("Reward")
        ax.set_xlabel("Interactions")

        if ax.get_legend() is None:
            scale = 0.65
            box1 = ax.get_position()
            ax.set_position([box1.x0, box1.y0 + box1.height * (1-scale), box1.width, box1.height * scale])
        else:
            ax.get_legend().remove()

        ax.legend(*ax.get_legend_handles_labels(), loc='upper left', bbox_to_anchor=(-.01, -.25), ncol=1, fontsize='medium') #type: ignore

        if show:
            plt.show()

        if filename:
            plt.savefig(filename, dpi=300)

    def __str__(self) -> str:
        return str({ "Learners": len(self._learners), "Environments": len(self._environments), "Interactions": len(self._interactions) })

    def __repr__(self) -> str:
        return str(self)


        templates = []
        for tname in [self.__class__.__name__]:
            templ = self.getRRDTemplateByName(tname)
            if templ: templates.append(templ)
        return templates

# This program is part of the HPMon Zenpack for Zenoss.
# You can find full information here: http://example.com

__doc__="""HPExpansionCard

HPExpansionCard is an abstraction of a PCI card.

$Id: HPExpansionCard.py,v 1.1 2010/06/29 10:40:17 egor Exp $"""

__version__ = "$Revision: 1.1 $"[11:-2]

from Products.ZenModel.ExpansionCard import *
from HPComponent import *

class HPExpansionCard(ExpansionCard, HPComponent):
    """ExpansionCard object"""
    slot = 0
    status = 1

    # we don't monitor cards
    monitor = False

    _properties = HWComponent._properties + (
        {'id':'slot', 'type':'int', 'mode':'w'},
        {'id':'status', 'type':'int', 'mode':'w'},
    )

    factory_type_information = (
        {
            'id'             : 'ExpansionCard',
            'meta_type'      : 'ExpansionCard',
            'description'    : """Arbitrary device grouping class""",
            'icon'           : 'ExpansionCard_icon.gif',
            'product'        : 'ZenModel',
            'factory'        : 'manage_addExpansionCard',
            'immediate_view' : 'viewHPExpansionCard',
            'actions'        :
            (
                { 'id'            : 'status'
                , 'name'          : 'Status'
                , 'action'        : 'viewHPExpansionCard'
                , 'permissions'   : (ZEN_VIEW,)
                },
                { 'id'            : 'viewHistory'
                , 'name'          : 'Modifications'
                , 'action'        : 'viewHistory'
                , 'permissions'   : (ZEN_VIEW_MODIFICATIONS,)
                },
            )
          },
        )

    def getRRDTemplates(self):
        """
        Return the RRD Templates list
        """


    time_diff = np.diff(csidata.timestamp_low)
    plt.figure(5)
    plt.plot(time_diff, linewidth=0.3, label='time diff')
    plt.legend()
    plt.title('time-diff')
    plt.xlabel('packets')
    plt.ylabel('time(us)')
    plt.show()

"""Plot csi of Linux 802.11n CSI Tool, observe csi quickly

Usage:
    python3 csishow.py ../material/5300/dataset/sample_0x1_ap.dat -t 1
"""
import argparse

import csiread
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import numpy as np
import scipy.signal as signal
from utils import scidx, calib, phy_ifft

def func_1(csidata):
    """CSI: time-amplitude"""
    s_index = 15    # subcarrier index
    csi = csidata.get_scaled_csi_sm()
    t = csidata.timestamp_low/1000000 - csidata.timestamp_low[0]/1000000

    amplitude = np.abs(csi[:, s_index])

    plt.figure()
    plt.plot(t, amplitude[:, 0, 0], linewidth=0.3, label='subcarrier_15_0_0')
    plt.plot(t, amplitude[:, 1, 0], linewidth=0.3, label='subcarrier_15_1_0')
    plt.plot(t, amplitude[:, 2, 0], linewidth=0.3, label='subcarrier_15_2_0')
    plt.legend()

    plt.title('csi-amplitude')
    plt.xlabel('time(s)')
    plt.ylabel('amplitude')
    plt.show()

def func_2(csidata):
    """CSI: subcarrier-amplitude(CFR)"""
    csi = csidata.get_scaled_csi_sm()
    amplitude = np.abs(csi)
    s_index = scidx(20, 2)

    plt.figure()
    plt.plot(s_index, amplitude[:100, :, 0, 0].T, 'r-', linewidth=0.3)
    plt.plot(s_index, amplitude[:100, :, 1, 0].T, 'g-', linewidth=0.3)
    plt.plot(s_index, amplitude[:100, :, 2, 0].T, 'y-', linewidth=0.3)

    patch_1 = mpatches.Patch(color='red', label=':100_r0t0')
    patch_2 = mpatches.Patch(color='green', label=':100_r1t0')
    patch_3 = mpatches.Patch(color='yellow', label=':100_r2t0')
    plt.legend(handles=[patch_1, patch_2, patch_3])

    plt.title('csi-amplitude')
    plt.xlabel('subcarriers')
    plt.ylabel('amplitude')
    plt.show()

def func_3(csidata):
    """CSI: time-phase"""
    s_index = 15    # subcarrier index
    csi = csidata.get_scaled_csi_sm()
    t = csidata.timestamp_low/1000000 - csidata.timestamp_low[0]/1000000
    phase = np.unwrap(np.angle(csi), axis=1)
    phase = calib(phase, scidx(20, 2))

    plt.figure()
    plt.plot(t, phase[:, s_index, 0, 0], linewidth=0.3, label='subcarrier_15_0_0')
    plt.plot(t, phase[:, s_index, 1, 0], linewidth=0.3, label='subcarrier_15_1_0')
    plt.plot(t, phase[:, s_index, 2, 0], linewidth=0.3, label='subcarrier_15_2_0')
    plt.legend()
    plt.title('csi-phase')
    plt.xlabel('time(s)')
    plt.ylabel('phase')
    plt.show()

def func_4(csidata):
    """CSI: subcarrier-phase"""
    csi = csidata.get_scaled_csi_sm()
    s_index = scidx(20, 2)
    phase = np.unwrap(np.angle(csi), axis=1)
    phase = calib(phase, s_index)

    plt.figure(4)
    plt.plot(s_index, phase[:100, :, 0, 0].T, 'r-', linewidth=0.3)
    plt.plot(s_index, phase[:100, :, 1, 0].T, 'g-', linewidth=0.3)
    plt.plot(s_index, phase[:100, :, 2, 0].T, 'y-', linewidth=0.3)

    patch_1 = mpatches.Patch(color='red', label=':100_r0t0')
    patch_2 = mpatches.Patch(color='green', label=':100_r1t0')
    patch_3 = mpatches.Patch(color='yellow', label=':100_r2t0')
    plt.legend(handles=[patch_1, patch_2, patch_3])

    plt.title('csi-phase')
    plt.xlabel('subcarriers')
    plt.ylabel('phase')
    plt.show()

def func_5(csidata):
    """timestamp_low: packet-time_difference

    Tips:
        Why are Intel.timestamp_low, Nexmon.sec and Nexmon.usec stored as
        ``np.uint32``?

        We may have such timestamp_low: ``[..., 2**32 - 2000, 2**32 - 1000,0,
        1000, 2000, ...]``. If timestamp_low.dtype is ``np.unint32``, we can get
        the correct time_diff by ``np.diff(csidata.timestamp_low).view(np.int32)``
        without any concern. When timestamp_low.dtype is ``np.int64``, the
        time_diff will contain negative values, we have to correct it by
        ``time_diff[time_diff < 0] += 2**32``.
    """

def func_6(csidata):
    """CSI: time-amplitude(CIR: OFDM symbol view)"""
    csi = csidata.get_scaled_csi_sm()

    s_index = scidx(20, 2)
    amplitude1 = np.abs(phy_ifft(csi[:100, :, 0, 0], s_index, axis=1)).T
    amplitude2 = np.abs(phy_ifft(csi[:100, :, 1, 0], s_index, axis=1)).T
    amplitude3 = np.abs(phy_ifft(csi[:100, :, 2, 0], s_index, axis=1)).T
    t = np.linspace(0, 64, 64)

    plt.figure(6)
    plt.plot(t, amplitude1, 'r-', linewidth=0.3)
    plt.plot(t, amplitude2, 'g-', linewidth=0.3)
    plt.plot(t, amplitude3, 'y-', linewidth=0.3)

    patch_1 = mpatches.Patch(color='red', label=':100_r0t0')
    patch_2 = mpatches.Patch(color='green', label=':100_r1t0')
    patch_3 = mpatches.Patch(color='yellow', label=':100_r2t0')
    plt.legend(handles=[patch_1, patch_2, patch_3])

    plt.title('csi-CIR')
    plt.xlabel('time(50ns)')
    plt.ylabel('amplitude')
    plt.show()

def func_8(csidata):
    """CSI: time-amplitude(butterworth filter)"""
    print("Waring: Please set parameters(butterworth) first")

    csi = csidata.get_scaled_csi_sm()[:1000]
    amplitude = np.abs(csi)
    t = (csidata.timestamp_low/1000000 - csidata.timestamp_low[0]/1000000)[:1000]

    sos = signal.butter(4, 100.0, 'lowpass', fs=900, output='sos')
    sfiltered = signal.sosfiltfilt(sos, amplitude, axis=0)

    plt.figure(9)
    plt.plot(t, amplitude[:, 15, 0, 0], '-', linewidth=1.5, alpha=0.25, label='origin_:1000s15r0t0')
    plt.plot(t, amplitude[:, 15, 1, 0], '-', linewidth=1.5, alpha=0.25, label='origin_:1000s15r1t0')
    plt.plot(t, amplitude[:, 15, 2, 0], '-', linewidth=1.5, alpha=0.25, label='origin_:1000s15r2t0')

    plt.plot(t, sfiltered[:, 15, 0, 0], '-', linewidth=1, label='buffer_:1000s15r0t0')
    plt.plot(t, sfiltered[:, 15, 1, 0], '-', linewidth=1, label='buffer_:1000s15r1t0')
    plt.plot(t, sfiltered[:, 15, 2, 0], '-', linewidth=1, label='buffer_:1000s15r2t0')

    plt.legend()
    plt.title("butterworth - lowpass")
    plt.xlabel('time(s)')
    plt.ylabel('amplitude')
    plt.show()

def func_9(csidata):
    index = 15
    csi = csidata.get_scaled_csi_sm()
    amplitude = np.abs(csi)
    t = (csidata.timestamp_low/1000000 - csidata.timestamp_low[0]/1000000)[:1000]

    kernel = np.array([[1], [2], [3], [2], [1]]) / 9
    amplitude1 = signal.convolve(amplitude[:1000, index:index+1, 0, 0], kernel, mode='same')
    amplitude2 = signal.convolve(amplitude[:1000, index:index+1, 1, 0], kernel, mode='same')
    amplitude3 = signal.convolve(amplitude[:1000, index:index+1, 2, 0], kernel, mode='same')

    plt.figure()
    plt.plot(t, amplitude1, 'r-', linewidth=0.3, label=':_s15r0t0')
    plt.plot(t, amplitude2, 'g-', linewidth=0.3, label=':_s15r1t0')
    plt.plot(t, amplitude3, 'y-', linewidth=0.3, label=':_s15r2t0')

    plt.legend()
    plt.title('csi amplitude conv smooth')
    plt.xlabel('time(s)')
    plt.ylabel('amplitude')
    plt.show()

helpinfo = r"""Plot Type

1   CSI-time-amplitude
2   CSI-subcarrier-amplitude
3   CSI-time-phase
4   CSI-subcarrier-phase
5   timestamp-packet-timediff
6   CSI-time-amplitude(CIR: OFDM symbol view)
8   CSI-time-amplitude(butterworth filter)
9   CSI-time-amplitude(Convolve)
"""

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('file', type=str, help='data file')
    parser.add_argument('-t', type=int, default=1, help=helpinfo)
    p = parser.parse_args()

    csidata = csiread.Intel(p.file, nrxnum=3, ntxnum=3)
    csidata.read()
    if p.t > 10:
        raise ValueError('the value of `t` can be 1 - 10')
    func = eval('func_' + str(p.t))
    func(csidata)


        func = get_function_definition(PACKAGE_NAME, FUNCTION_NAME)
        assert func is not None

"""Tests using pytest_resilient_circuits"""

import pytest
from resilient_circuits.util import get_config_data, get_function_definition
from resilient_circuits import SubmitTestFunction, FunctionResult

PACKAGE_NAME = "fn_main_mock_integration"
FUNCTION_NAME = "mock_function_two"

# Read the default configuration-data section from the package
config_data = get_config_data(PACKAGE_NAME)

# Provide a simulation of the Resilient REST API (uncomment to connect to a real appliance)
resilient_mock = "pytest_resilient_circuits.BasicResilientMock"

def call_mock_function_two_function(circuits, function_params, timeout=5):
    # Create the submitTestFunction event
    evt = SubmitTestFunction("mock_function_two", function_params)

    # Fire a message to the function
    circuits.manager.fire(evt)

    # circuits will fire an "exception" event if an exception is raised in the FunctionComponent
    exception_event = circuits.watcher.wait("exception", parent=None, timeout=timeout)

    if exception_event is not False:
        exception = exception_event.args[1].args[1]
        raise exception

    # else return the FunctionComponent's results
    else:
        event = circuits.watcher.wait("mock_function_two_result", parent=evt, timeout=timeout)
        assert event
        assert isinstance(event.kwargs["result"], FunctionResult)
        pytest.wait_for(event, "complete", True)
        return event.kwargs["result"].value

class TestMockFunctionTwo:
    """ Tests for the mock_function_two function"""

    def test_function_definition(self):
        """ Test that the package provides customization_data that defines the function """

    mock_inputs_1 = {
        "mock_input_text": "sample text",
        "mock_input_number": 123,
        "mock_input_boolean": True
    }

    expected_results_1 = {"value": "xyz"}

    mock_inputs_2 = {
        "mock_input_text": "sample text",
        "mock_input_number": 123,
        "mock_input_boolean": True
    }

    expected_results_2 = {"value": "xyz"}

    @pytest.mark.parametrize("mock_inputs, expected_results", [
        (mock_inputs_1, expected_results_1),
        (mock_inputs_2, expected_results_2)
    ])
    def test_success(self, circuits_app, mock_inputs, expected_results):
        """ Test calling with sample values for the parameters """

        results = call_mock_function_two_function(circuits_app, mock_inputs)
        assert(expected_results == results)