    for d in drives:
        d.set_distance()

"""
Created on Sun Jul 07 16:48:06 2013

@author: dima
"""

import datareader as dr
import overlap as ol
import plotter as pl

def calculate_overlaps(drives, dist_tol, time_tol):
    """
    for each drive, determines and sets Overlap with all other drives
    """

    for i1 in range(len(drives)-1):
        d1 = drives[i1]

        for i2 in range(i1+1, len(drives)):
            d2 = drives[i2]

            #stop trying if d1 ends more than time_tol before d2 starts
            #note that drives are chronologically ordered
            if d2.coords[0].time - d1.coords[-1].time > time_tol:
                break

            overlap = ol.compute_overlap(d1, d2, dist_tol, time_tol)
            if overlap:
                ol1 = ol.Overlap(d1, d2, overlap[0], overlap[1])
                d1.append_overlap(ol1)
                ol2 = ol.Overlap(d2, d1, overlap[2], overlap[3])
                d2.append_overlap(ol2)

def calculate_distances(drives):
    """
    for each drive, determines and sets distance
    """

def main():
    drives = dr.read_data(open("gpsdata/all.tsv"), 1000)
    calculate_distances(drives)
    calculate_overlaps(drives, 0.01, 900)
    pl.distance_plot(drives)
    pl.max_overlap_fractions_plot(drives)

main()


  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import os.path as osp
import numpy as np
# `pip install easydict` if you don't have it
from easydict import EasyDict as edict

__C = edict()
# Consumers can get config by:
cfg = __C

# Training options
__C.TRAIN = edict()

# Initial learning rate
__C.TRAIN.LEARNING_RATE = 0.001

# Momentum
__C.TRAIN.MOMENTUM = 0.9

# Weight decay, for regularization
__C.TRAIN.WEIGHT_DECAY = 0.0001

# Factor for reducing the learning rate
__C.TRAIN.GAMMA = 0.1

# Use uncertainty to weight individual loss
__C.TRAIN.MULTI_TASK_LOSS = True

# Use horizontally-flipped images during training?
__C.TRAIN.USE_FLIPPED = True

# Whether to double the learning rate for bias
__C.TRAIN.DOUBLE_BIAS = False

# Whether to initialize the weights with truncated normal distribution
__C.TRAIN.TRUNCATED = False

# Whether to have weight decay on bias as well
__C.TRAIN.BIAS_DECAY = False

# The scale is the pixel size of an image's shortest side
__C.TRAIN.SCALES = (600,)

# Max pixel size of the longest side of a scaled input image
__C.TRAIN.MAX_SIZE = 2484

# Minibatch size (number of regions of interest [ROIs])
__C.TRAIN.BATCH_SIZE = 512

# Fraction of minibatch that is labeled foreground (i.e. class > 0)
__C.TRAIN.FG_FRACTION = 0.25

# Overlap threshold for a ROI to be considered foreground (if >= FG_THRESH)
__C.TRAIN.FG_THRESH = 0.5

# Overlap threshold for a ROI to be considered background (class = 0 if
__C.TRAIN.BG_THRESH_HI = 0.5
__C.TRAIN.BG_THRESH_LO = 0.0

# Normalize the targets (subtract empirical mean, divide by empirical stddev)
__C.TRAIN.BBOX_NORMALIZE_TARGETS = True

__C.TRAIN.BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)

# Normalize the targets using "precomputed" (or made up) means and stdevs
# (BBOX_NORMALIZE_TARGETS must also be True)
__C.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED = True
__C.TRAIN.BBOX_NORMALIZE_MEANS = (0.0, 0.0, 0.0, 0.0)
__C.TRAIN.BBOX_NORMALIZE_STDS = (0.1, 0.1, 0.2, 0.2)

__C.TRAIN.DIM_NORMALIZE_TARGETS_PRECOMPUTED = True
__C.TRAIN.DIM_NORMALIZE_MEANS = (1.6, 1.5, 4.0, 0.0, 0.0)
__C.TRAIN.DIM_NORMALIZE_STDS = (0.5, 0.5, 0.5, 0.5, 0.5)

# IOU >= thresh: positive example
__C.TRAIN.RPN_POSITIVE_OVERLAP = 0.7
# IOU < thresh: negative example
__C.TRAIN.RPN_NEGATIVE_OVERLAP = 0.3
# If an anchor statisfied by positive and negative conditions set to negative
__C.TRAIN.RPN_CLOBBER_POSITIVES = False
# Max number of foreground examples
__C.TRAIN.RPN_FG_FRACTION = 0.5
# Total number of examples
__C.TRAIN.RPN_BATCHSIZE = 512
# NMS threshold used on RPN proposals
__C.TRAIN.RPN_NMS_THRESH = 0.7
# Number of top scoring boxes to keep before apply NMS to RPN proposals
__C.TRAIN.RPN_PRE_NMS_TOP_N = 12000
# Number of top scoring boxes to keep after applying NMS to RPN proposals
__C.TRAIN.RPN_POST_NMS_TOP_N = 2000
# Proposal height and width both need to be greater than RPN_MIN_SIZE (at orig image scale)
__C.TRAIN.RPN_MIN_SIZE = 8
# Deprecated (outside weights)
__C.TRAIN.RPN_BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)
# Give the positive RPN examples weight of p * 1 / {num positives}
# and give negatives a weight of (1 - p)
# Set to -1.0 to use uniform example weighting
__C.TRAIN.RPN_POSITIVE_WEIGHT = -1.0

# Testing options
__C.TEST = edict()

# Scale to use during testing (can NOT list multiple scales)
# The scale is the pixel size of an image's shortest side
__C.TEST.SCALES = (600,)

# Max pixel size of the longest side of a scaled input image
__C.TEST.MAX_SIZE = 2484

# Overlap threshold used for non-maximum suppression (suppress boxes with
__C.TEST.NMS = 0.3

## NMS threshold used on RPN proposals
__C.TEST.RPN_NMS_THRESH = 0.7
## Number of top scoring boxes to keep before apply NMS to RPN proposals
__C.TEST.RPN_PRE_NMS_TOP_N = 6000

## Number of top scoring boxes to keep after applying NMS to RPN proposals
__C.TEST.RPN_POST_NMS_TOP_N = 300

# Proposal height and width both need to be greater than RPN_MIN_SIZE (at orig image scale)
__C.TEST.RPN_MIN_SIZE = 16

# Only useful when TEST.MODE is 'top', specifies the number of top proposals to select
__C.TEST.RPN_TOP_N = 5000

# ResNet options

__C.RESNET = edict()

# Option to set if max-pooling is appended after crop_and_resize.
# if true, the region will be resized to a square of 2xPOOLING_SIZE,
# then 2x2 max-pooling is applied; otherwise the region will be directly
# resized to a square of POOLING_SIZE
__C.RESNET.MAX_POOL = False

# Number of fixed blocks during training, by default the first of all 4 blocks is fixed
__C.RESNET.FIXED_BLOCKS = 1

# MISC

# The mapping from image coordinates to feature map coordinates might cause
# some boxes that are distinct in image space to become identical in feature
# coordinates. If DEDUP_BOXES > 0, then DEDUP_BOXES is used as the scale factor
# for identifying duplicate boxes.
__C.DEDUP_BOXES = 1. / 16.

# Pixel mean values (BGR order) as a (1, 1, 3) array
# We use the same pixel mean for all networks even though it's not exactly what
# they were trained with
__C.PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])

# ROI size of for the keypoint branch
__C.KPTS_GRID = 28

__C.DIM_MEANS = np.array([[[1.64, 1.52, 3.875]]])

# For reproducibility
__C.RNG_SEED = 3

# A small number that's used many times
__C.EPS = 1e-14

# Root directory of project
__C.ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..', '..'))

# Data directory
__C.DATA_DIR = osp.abspath(osp.join(__C.ROOT_DIR, 'data'))

# Name (or path to) the matlab executable
__C.MATLAB = 'matlab'

# Place outputs under an experiments directory
__C.EXP_DIR = 'res101'

# Use GPU implementation of non-maximum suppression
__C.USE_GPU_NMS = True

# Default GPU device id
__C.GPU_ID = 0

__C.POOLING_MODE = 'align'

# Size of the pooled region after RoI pooling
__C.POOLING_SIZE = 7

# Maximal number of gt boxes in an image during training
__C.MAX_NUM_GT_BOXES = 30

# Anchor ratios for RPN
__C.ANCHOR_RATIOS = [0.5,1,2]

# Feature stride for RPN
__C.FEAT_STRIDE = [16, ]

# Anchor scales for RPN_FPN
__C.FPN_ANCHOR_SCALES = [32, 64, 128, 256, 512]

# Feature stride for RPN_FPN
__C.FPN_FEAT_STRIDES = [4, 8, 16, 32, 64]

# Anchor stride for RPN_FPN
__C.FPN_ANCHOR_STRIDE = 1

import pdb
def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.
  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.
  A canonical path is built using the name from an imdb and a network
  (if not None).
  """

def _merge_a_into_b(a, b):
  """Merge config dictionary a into config dictionary b, clobbering the
  options in b whenever they are also specified in a.
  """
  if type(a) is not edict:
    return

  for k, v in a.items():
    # a must specify keys that are in b
    if k not in b:
      raise KeyError('{} is not a valid config key'.format(k))

    # the types must match, too
    old_type = type(b[k])
    if old_type is not type(v):
      if isinstance(b[k], np.ndarray):
        v = np.array(v, dtype=b[k].dtype)
      else:
        raise ValueError(('Type mismatch ({} vs. {}) '
                          'for config key: {}').format(type(b[k]),
                                                       type(v), k))

    # recursively merge dicts
    if type(v) is edict:
      try:
        _merge_a_into_b(a[k], b[k])
      except:
        print(('Error under config key: {}'.format(k)))
        raise
    else:
      b[k] = v

def cfg_from_file(filename):
  """Load a config file and merge it into the default options."""
  import yaml
  with open(filename, 'r') as f:
    yaml_cfg = edict(yaml.load(f))

  _merge_a_into_b(yaml_cfg, __C)

def cfg_from_list(cfg_list):
  """Set config keys via list (e.g., from command line)."""
  from ast import literal_eval
  assert len(cfg_list) % 2 == 0
  for k, v in zip(cfg_list[0::2], cfg_list[1::2]):
    key_list = k.split('.')
    d = __C
    for subkey in key_list[:-1]:
      assert subkey in d
      d = d[subkey]
    subkey = key_list[-1]
    assert subkey in d
    try:
      value = literal_eval(v)
    except:
      # handle the case when v is a string literal
      value = v
    assert type(value) == type(d[subkey]), \
      'type {} does not match original type {}'.format(
        type(value), type(d[subkey]))
    d[subkey] = value


        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is not None:
            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1]

"""Tokenization classes for XLM."""

import json
import os
import re
import sys
import unicodedata
from typing import List, Optional, Tuple

import sacremoses as sm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging

logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "xlm-mlm-en-2048": "http://example.com",
        "xlm-mlm-ende-1024": "http://example.com",
        "xlm-mlm-enfr-1024": "http://example.com",
        "xlm-mlm-enro-1024": "http://example.com",
        "xlm-mlm-tlm-xnli15-1024": "http://example.com",
        "xlm-mlm-xnli15-1024": "http://example.com",
        "xlm-clm-enfr-1024": "http://example.com",
        "xlm-clm-ende-1024": "http://example.com",
        "xlm-mlm-17-1280": "http://example.com",
        "xlm-mlm-100-1280": "http://example.com",
    },
    "merges_file": {
        "xlm-mlm-en-2048": "http://example.com",
        "xlm-mlm-ende-1024": "http://example.com",
        "xlm-mlm-enfr-1024": "http://example.com",
        "xlm-mlm-enro-1024": "http://example.com",
        "xlm-mlm-tlm-xnli15-1024": "http://example.com",
        "xlm-mlm-xnli15-1024": "http://example.com",
        "xlm-clm-enfr-1024": "http://example.com",
        "xlm-clm-ende-1024": "http://example.com",
        "xlm-mlm-17-1280": "http://example.com",
        "xlm-mlm-100-1280": "http://example.com",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "xlm-mlm-en-2048": 512,
    "xlm-mlm-ende-1024": 512,
    "xlm-mlm-enfr-1024": 512,
    "xlm-mlm-enro-1024": 512,
    "xlm-mlm-tlm-xnli15-1024": 512,
    "xlm-mlm-xnli15-1024": 512,
    "xlm-clm-enfr-1024": 512,
    "xlm-clm-ende-1024": 512,
    "xlm-mlm-17-1280": 512,
    "xlm-mlm-100-1280": 512,
}

PRETRAINED_INIT_CONFIGURATION = {
    "xlm-mlm-en-2048": {"do_lowercase_and_remove_accent": True},
    "xlm-mlm-ende-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "de", 1: "en"},
        "lang2id": {"de": 0, "en": 1},
    },
    "xlm-mlm-enfr-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "fr"},
        "lang2id": {"en": 0, "fr": 1},
    },
    "xlm-mlm-enro-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "ro"},
        "lang2id": {"en": 0, "ro": 1},
    },
    "xlm-mlm-tlm-xnli15-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {
            0: "ar",
            1: "bg",
            2: "de",
            3: "el",
            4: "en",
            5: "es",
            6: "fr",
            7: "hi",
            8: "ru",
            9: "sw",
            10: "th",
            11: "tr",
            12: "ur",
            13: "vi",
            14: "zh",
        },
        "lang2id": {
            "ar": 0,
            "bg": 1,
            "de": 2,
            "el": 3,
            "en": 4,
            "es": 5,
            "fr": 6,
            "hi": 7,
            "ru": 8,
            "sw": 9,
            "th": 10,
            "tr": 11,
            "ur": 12,
            "vi": 13,
            "zh": 14,
        },
    },
    "xlm-mlm-xnli15-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {
            0: "ar",
            1: "bg",
            2: "de",
            3: "el",
            4: "en",
            5: "es",
            6: "fr",
            7: "hi",
            8: "ru",
            9: "sw",
            10: "th",
            11: "tr",
            12: "ur",
            13: "vi",
            14: "zh",
        },
        "lang2id": {
            "ar": 0,
            "bg": 1,
            "de": 2,
            "el": 3,
            "en": 4,
            "es": 5,
            "fr": 6,
            "hi": 7,
            "ru": 8,
            "sw": 9,
            "th": 10,
            "tr": 11,
            "ur": 12,
            "vi": 13,
            "zh": 14,
        },
    },
    "xlm-clm-enfr-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "fr"},
        "lang2id": {"en": 0, "fr": 1},
    },
    "xlm-clm-ende-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "de", 1: "en"},
        "lang2id": {"de": 0, "en": 1},
    },
    "xlm-mlm-17-1280": {
        "do_lowercase_and_remove_accent": False,
        "id2lang": {
            0: "ar",
            1: "de",
            2: "en",
            3: "es",
            4: "fr",
            5: "hi",
            6: "it",
            7: "ja",
            8: "ko",
            9: "nl",
            10: "pl",
            11: "pt",
            12: "ru",
            13: "sv",
            14: "tr",
            15: "vi",
            16: "zh",
        },
        "lang2id": {
            "ar": 0,
            "de": 1,
            "en": 2,
            "es": 3,
            "fr": 4,
            "hi": 5,
            "it": 6,
            "ja": 7,
            "ko": 8,
            "nl": 9,
            "pl": 10,
            "pt": 11,
            "ru": 12,
            "sv": 13,
            "tr": 14,
            "vi": 15,
            "zh": 16,
        },
    },
    "xlm-mlm-100-1280": {
        "do_lowercase_and_remove_accent": False,
        "id2lang": {
            0: "af",
            1: "als",
            2: "am",
            3: "an",
            4: "ang",
            5: "ar",
            6: "arz",
            7: "ast",
            8: "az",
            9: "bar",
            10: "be",
            11: "bg",
            12: "bn",
            13: "br",
            14: "bs",
            15: "ca",
            16: "ceb",
            17: "ckb",
            18: "cs",
            19: "cy",
            20: "da",
            21: "de",
            22: "el",
            23: "en",
            24: "eo",
            25: "es",
            26: "et",
            27: "eu",
            28: "fa",
            29: "fi",
            30: "fr",
            31: "fy",
            32: "ga",
            33: "gan",
            34: "gl",
            35: "gu",
            36: "he",
            37: "hi",
            38: "hr",
            39: "hu",
            40: "hy",
            41: "ia",
            42: "id",
            43: "is",
            44: "it",
            45: "ja",
            46: "jv",
            47: "ka",
            48: "kk",
            49: "kn",
            50: "ko",
            51: "ku",
            52: "la",
            53: "lb",
            54: "lt",
            55: "lv",
            56: "mk",
            57: "ml",
            58: "mn",
            59: "mr",
            60: "ms",
            61: "my",
            62: "nds",
            63: "ne",
            64: "nl",
            65: "nn",
            66: "no",
            67: "oc",
            68: "pl",
            69: "pt",
            70: "ro",
            71: "ru",
            72: "scn",
            73: "sco",
            74: "sh",
            75: "si",
            76: "simple",
            77: "sk",
            78: "sl",
            79: "sq",
            80: "sr",
            81: "sv",
            82: "sw",
            83: "ta",
            84: "te",
            85: "th",
            86: "tl",
            87: "tr",
            88: "tt",
            89: "uk",
            90: "ur",
            91: "uz",
            92: "vi",
            93: "war",
            94: "wuu",
            95: "yi",
            96: "zh",
            97: "zh_classical",
            98: "zh_min_nan",
            99: "zh_yue",
        },
        "lang2id": {
            "af": 0,
            "als": 1,
            "am": 2,
            "an": 3,
            "ang": 4,
            "ar": 5,
            "arz": 6,
            "ast": 7,
            "az": 8,
            "bar": 9,
            "be": 10,
            "bg": 11,
            "bn": 12,
            "br": 13,
            "bs": 14,
            "ca": 15,
            "ceb": 16,
            "ckb": 17,
            "cs": 18,
            "cy": 19,
            "da": 20,
            "de": 21,
            "el": 22,
            "en": 23,
            "eo": 24,
            "es": 25,
            "et": 26,
            "eu": 27,
            "fa": 28,
            "fi": 29,
            "fr": 30,
            "fy": 31,
            "ga": 32,
            "gan": 33,
            "gl": 34,
            "gu": 35,
            "he": 36,
            "hi": 37,
            "hr": 38,
            "hu": 39,
            "hy": 40,
            "ia": 41,
            "id": 42,
            "is": 43,
            "it": 44,
            "ja": 45,
            "jv": 46,
            "ka": 47,
            "kk": 48,
            "kn": 49,
            "ko": 50,
            "ku": 51,
            "la": 52,
            "lb": 53,
            "lt": 54,
            "lv": 55,
            "mk": 56,
            "ml": 57,
            "mn": 58,
            "mr": 59,
            "ms": 60,
            "my": 61,
            "nds": 62,
            "ne": 63,
            "nl": 64,
            "nn": 65,
            "no": 66,
            "oc": 67,
            "pl": 68,
            "pt": 69,
            "ro": 70,
            "ru": 71,
            "scn": 72,
            "sco": 73,
            "sh": 74,
            "si": 75,
            "simple": 76,
            "sk": 77,
            "sl": 78,
            "sq": 79,
            "sr": 80,
            "sv": 81,
            "sw": 82,
            "ta": 83,
            "te": 84,
            "th": 85,
            "tl": 86,
            "tr": 87,
            "tt": 88,
            "uk": 89,
            "ur": 90,
            "uz": 91,
            "vi": 92,
            "war": 93,
            "wuu": 94,
            "yi": 95,
            "zh": 96,
            "zh_classical": 97,
            "zh_min_nan": 98,
            "zh_yue": 99,
        },
    },
}

def get_pairs(word):
    """
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def lowercase_and_remove_accent(text):
    """
    Lowercase and strips accents from a piece of text based on
    http://example.com
    """
    text = " ".join(text)
    text = text.lower()
    text = unicodedata.normalize("NFD", text)
    output = []
    for char in text:
        cat = unicodedata.category(char)
        if cat == "Mn":
            continue
        output.append(char)
    return "".join(output).lower().split(" ")

def replace_unicode_punct(text):
    """
    Port of http://example.com
    """
    text = text.replace("，", ",")
    text = re.sub(r"。\s*", ". ", text)
    text = text.replace("、", ",")
    text = text.replace("”", '"')
    text = text.replace("“", '"')
    text = text.replace("∶", ":")
    text = text.replace("：", ":")
    text = text.replace("？", "?")
    text = text.replace("《", '"')
    text = text.replace("》", '"')
    text = text.replace("）", ")")
    text = text.replace("！", "!")
    text = text.replace("（", "(")
    text = text.replace("；", ";")
    text = text.replace("１", "1")
    text = text.replace("」", '"')
    text = text.replace("「", '"')
    text = text.replace("０", "0")
    text = text.replace("３", "3")
    text = text.replace("２", "2")
    text = text.replace("５", "5")
    text = text.replace("６", "6")
    text = text.replace("９", "9")
    text = text.replace("７", "7")
    text = text.replace("８", "8")
    text = text.replace("４", "4")
    text = re.sub(r"．\s*", ". ", text)
    text = text.replace("～", "~")
    text = text.replace("’", "'")
    text = text.replace("…", "...")
    text = text.replace("━", "-")
    text = text.replace("〈", "<")
    text = text.replace("〉", ">")
    text = text.replace("【", "[")
    text = text.replace("】", "]")
    text = text.replace("％", "%")
    return text

def remove_non_printing_char(text):
    """
    Port of http://example.com
    """
    output = []
    for char in text:
        cat = unicodedata.category(char)
        if cat.startswith("C"):
            continue
        output.append(char)
    return "".join(output)

def romanian_preprocessing(text):
    """Sennrich's WMT16 scripts for Romanian preprocessing, used by model `xlm-mlm-enro-1024`"""
    text = text.replace("\u015e", "\u0218").replace("\u015f", "\u0219")
    text = text.replace("\u0162", "\u021a").replace("\u0163", "\u021b")
    text = text.replace("\u0218", "S").replace("\u0219", "s")  # s-comma
    text = text.replace("\u021a", "T").replace("\u021b", "t")  # t-comma
    text = text.replace("\u0102", "A").replace("\u0103", "a")
    text = text.replace("\u00C2", "A").replace("\u00E2", "a")
    text = text.replace("\u00CE", "I").replace("\u00EE", "i")
    return text

class XLMTokenizer(PreTrainedTokenizer):
    """
    Construct an XLM tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:

    - Moses preprocessing and tokenization for most supported languages.
    - Language specific tokenization for Chinese (Jieba), Japanese (KyTea) and Thai (PyThaiNLP).
    - Optionally lowercases and normalizes all inputs text.
    - The arguments ``special_tokens`` and the function ``set_special_tokens``, can be used to add additional symbols
      (like "__classify__") to a vocabulary.
    - The :obj:`lang2id` attribute maps the languages supported by the model with their IDs if provided (automatically
      set for pretrained vocabularies).
    - The :obj:`id2lang` attributes does reverse mapping if provided (automatically set for pretrained vocabularies).

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods.
    Users should refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (:obj:`str`):
            Vocabulary file.
        merges_file (:obj:`str`):
            Merges file.
        unk_token (:obj:`str`, `optional`, defaults to :obj:`"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            .. note::

                When building a sequence using special tokens, this is not the token that is used for the beginning of
                sequence. The token used is the :obj:`cls_token`.
        sep_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (:obj:`str`, `optional`, defaults to :obj:`"<special1>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        additional_special_tokens (:obj:`List[str]`, `optional`, defaults to :obj:`["<special0>","<special1>","<special2>","<special3>","<special4>","<special5>","<special6>","<special7>","<special8>","<special9>"]`):
            List of additional special tokens.
        lang2id (:obj:`Dict[str, int]`, `optional`):
            Dictionary mapping languages string identifiers to their IDs.
        id2lang (:obj:`Dict[int, str]`, `optional`):
            Dictionary mapping language IDs to their string identifiers.
        do_lowercase_and_remove_accent (:obj:`bool`, `optional`, defaults to :obj:`True`):
            Whether to lowercase and remove accents when tokenizing.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(
        self,
        vocab_file,
        merges_file,
        unk_token="<unk>",
        bos_token="<s>",
        sep_token="</s>",
        pad_token="<pad>",
        cls_token="</s>",
        mask_token="<special1>",
        additional_special_tokens=[
            "<special0>",
            "<special1>",
            "<special2>",
            "<special3>",
            "<special4>",
            "<special5>",
            "<special6>",
            "<special7>",
            "<special8>",
            "<special9>",
        ],
        lang2id=None,
        id2lang=None,
        do_lowercase_and_remove_accent=True,
        **kwargs
    ):
        super().__init__(
            unk_token=unk_token,
            bos_token=bos_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            additional_special_tokens=additional_special_tokens,
            lang2id=lang2id,
            id2lang=id2lang,
            do_lowercase_and_remove_accent=do_lowercase_and_remove_accent,
            **kwargs,
        )

        # cache of sm.MosesPunctNormalizer instance
        self.cache_moses_punct_normalizer = dict()
        # cache of sm.MosesTokenizer instance
        self.cache_moses_tokenizer = dict()
        self.lang_with_custom_tokenizer = set(["zh", "th", "ja"])
        # True for current supported model (v1.2.0), False for XLM-17 & 100
        self.do_lowercase_and_remove_accent = do_lowercase_and_remove_accent
        self.lang2id = lang2id
        self.id2lang = id2lang
        if lang2id is not None and id2lang is not None:
            assert len(lang2id) == len(id2lang)

        self.ja_word_tokenizer = None
        self.zh_word_tokenizer = None

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[:-1]
        merges = [tuple(merge.split()[:2]) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}

    @property
    def do_lower_case(self):
        return self.do_lowercase_and_remove_accent

    def moses_punct_norm(self, text, lang):
        if lang not in self.cache_moses_punct_normalizer:
            punct_normalizer = sm.MosesPunctNormalizer(lang=lang)
            self.cache_moses_punct_normalizer[lang] = punct_normalizer
        else:
            punct_normalizer = self.cache_moses_punct_normalizer[lang]
        return punct_normalizer.normalize(text)

    def moses_tokenize(self, text, lang):
        if lang not in self.cache_moses_tokenizer:
            moses_tokenizer = sm.MosesTokenizer(lang=lang)
            self.cache_moses_tokenizer[lang] = moses_tokenizer
        else:
            moses_tokenizer = self.cache_moses_tokenizer[lang]
        return moses_tokenizer.tokenize(text, return_str=False, escape=False)

    def moses_pipeline(self, text, lang):
        text = replace_unicode_punct(text)
        text = self.moses_punct_norm(text, lang)
        text = remove_non_printing_char(text)
        return text

    def ja_tokenize(self, text):
        if self.ja_word_tokenizer is None:
            try:
                import Mykytea

                self.ja_word_tokenizer = Mykytea.Mykytea(
                    f"-model {os.path.expanduser('~')}/local/share/kytea/model.bin"
                )
            except (AttributeError, ImportError):
                logger.error(
                    "Make sure you install KyTea (http://example.com) and it's python wrapper (http://example.com) with the following steps"
                )
                logger.error("1. git clone http://example.com && cd kytea")
                logger.error("2. autoreconf -i")
                logger.error("3. ./configure --prefix=$HOME/local")
                logger.error("4. make && make install")
                logger.error("5. pip install kytea")
                raise
        return list(self.ja_word_tokenizer.getWS(text))

    @property
    def vocab_size(self):
        return len(self.encoder)

    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
        self.cache[token] = word
        return word

    def _tokenize(self, text, lang="en", bypass_tokenizer=False):
        """
        Tokenize a string given language code. For Chinese, Japanese and Thai, we use a language specific tokenizer.
        Otherwise, we use Moses.

        Details of tokenization:

            - [sacremoses](http://example.com): port of Moses
            - Install with `pip install sacremoses`
            - [pythainlp](http://example.com): Thai tokenizer
            - Install with `pip install pythainlp`
            - [kytea](https://github.com/chezou/Mykytea-python): Japanese tokenizer, wrapper of
              [KyTea](https://github.com/neubig/kytea)
            - Install with the following steps:

            ::

                git clone git@github.com:neubig/kytea.git && cd kytea
                autoreconf -i
                ./configure --prefix=$HOME/local
                make && make install
                pip install kytea

            - [jieba](http://example.com): Chinese tokenizer (*)
            - Install with `pip install jieba`

        (*) The original XLM used [Stanford
        Segmenter](http://example.com). However, the wrapper
        (`nltk.tokenize.stanford_segmenter`) is slow due to JVM overhead, and it will be deprecated. Jieba is a lot
        faster and pip-installable. Note there is some mismatch with the Stanford Segmenter. It should be fine if you
        fine-tune the model with Chinese supervisionself. If you want the same exact behaviour, use the original XLM
        [preprocessing script](http://example.com) to tokenize the sentence
        externally, and set `bypass_tokenizer=True` to bypass the tokenizer.

        Args:

            - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported
              languages. However, we don't enforce it.
            - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)
              (bool). If True, we only apply BPE.

        Returns:
            List of tokens.
        """
        if lang and self.lang2id and lang not in self.lang2id:
            logger.error(
                "Supplied language code not found in lang2id mapping. Please check that your language is supported by the loaded pretrained model."
            )
        if bypass_tokenizer:
            text = text.split()
        elif lang not in self.lang_with_custom_tokenizer:
            text = self.moses_pipeline(text, lang=lang)
            # TODO: make sure we are using `xlm-mlm-enro-1024`, since XLM-100 doesn't have this step
            if lang == "ro":
                text = romanian_preprocessing(text)
            text = self.moses_tokenize(text, lang=lang)
        elif lang == "th":
            text = self.moses_pipeline(text, lang=lang)
            try:
                if "pythainlp" not in sys.modules:
                    from pythainlp.tokenize import word_tokenize as th_word_tokenize
                else:
                    th_word_tokenize = sys.modules["pythainlp"].word_tokenize
            except (AttributeError, ImportError):
                logger.error(
                    "Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps"
                )
                logger.error("1. pip install pythainlp")
                raise
            text = th_word_tokenize(text)
        elif lang == "zh":
            try:
                if "jieba" not in sys.modules:
                    import jieba
                else:
                    jieba = sys.modules["jieba"]
            except (AttributeError, ImportError):
                logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
                logger.error("1. pip install jieba")
                raise
            text = " ".join(jieba.cut(text))
            text = self.moses_pipeline(text, lang=lang)
            text = text.split()
        elif lang == "ja":
            text = self.moses_pipeline(text, lang=lang)
            text = self.ja_tokenize(text)
        else:
            raise ValueError("It should not reach here")

        if self.do_lowercase_and_remove_accent and not bypass_tokenizer:
            text = lowercase_and_remove_accent(text)

        split_tokens = []
        for token in text:
            if token:
                split_tokens.extend([t for t in self.bpe(token).split(" ")])

        return split_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        out_string = "".join(tokens).replace("</w>", " ").strip()
        return out_string

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An XLM sequence has the following format:

        - single sequence: ``<s> X </s>``
        - pair of sequences: ``<s> A </s> B </s>``

        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (:obj:`List[int]`, `optional`):
                Optional second list of IDs for sequence pairs.

        Returns:
            :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.

        """
        bos = [self.bos_token_id]
        sep = [self.sep_token_id]

        if token_ids_1 is None:
            return bos + token_ids_0 + sep
        return bos + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer ``prepare_for_model`` method.

        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs.
            token_ids_1 (:obj:`List[int]`, `optional`):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence
        pair mask has the following format:

        ::

            0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
            | first sequence    | second sequence |

        If :obj:`token_ids_1` is :obj:`None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs.
            token_ids_1 (:obj:`List[int]`, `optional`):
                Optional second list of IDs for sequence pairs.

        Returns:
            :obj:`List[int]`: List of `token type IDs <../glossary.html#token-type-ids>`_ according to the given
            sequence(s).
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, ensure_ascii=False))

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        return vocab_file, merge_file


while True:
            match = re.search('[0-9]{15}',str(lines)[start:])
            if match is not None:
                if str(lines)[start + match.start():start + match.end()] not in aux:
                    aux.append(str(lines)[start + match.start():start + match.end()])
                start += match.end()
            else:
                break;

from HTMLParser import HTMLParser
import simplejson as json
import database
import re

class MyHTMLParser(HTMLParser):
    def array(self):
        self.dataArray = []
    def handle_data(self, data):
        self.dataArray.append(data)

def htmlFormat(json_dump):
    html = "
 Name: %s - Link: facebook profile - Gender: %s -Locale: %s
" %(json_dump['username'],json_dump['name'], json_dump['link'], json_dump['gender'], json_dump['locale'])
    return html

def parceros(json_dump):
    parser = MyHTMLParser()
    parser.array()
    names = []
    userIds = []
    try:
        to_parse = str(json_dump['domops'][0][3]['__html'])
        parser.feed(to_parse)
    except:
        print 'Error in json dump or parser.feed'
    i = 0
    while True:
        if ((parser.dataArray[i] == 'Test Users') or (parser.dataArray[i] == 'Delete') or (parser.dataArray[i] == 'Add') or
            (parser.dataArray[i] == 'Name') or (parser.dataArray[i] == 'User ID') or (parser.dataArray[i] == 'Email') or
            (parser.dataArray[i] == 'Edit') or (parser.dataArray[i] == 'tfbnw.net')):
            del parser.dataArray[i]
        else:
            i += 1
        if i == len(parser.dataArray):
            break

    i = 0

    while i < (len(parser.dataArray) - 2):
        names.append(parser.dataArray[i])
        userIds.append(parser.dataArray[i+1])
        i = i + 3

    if ( userIds!=[] and names!=[]) and (parser.dataArray[0] != 'This app has no Test Users.'):
        database.insertTestUsersDev(userIds,names)
        return 1
    else:
        return -1

def parseData(dataRaw):
    parser = MyHTMLParser()
    parser.array()
    names = []
    userIds = []
    emails = []
    passwords = []

    for data in dataRaw:
        if data=="":
            continue
        text = data.strip("for (;;);")
        json_dump = json.loads(text)
        try:
            to_parse = str(json_dump['jsmods']['markup'][0][1]['__html'])
            parser.feed(to_parse)
        except:
            print 'Error in json dump or parser.feed'

    for i in range(len(parser.dataArray)):
        if parser.dataArray[i] == 'Name':
            names.append(parser.dataArray[i+1])
            continue
        if parser.dataArray[i] == 'User ID':
            userIds.append(parser.dataArray[i+1])
            continue
        if parser.dataArray[i] == 'Login email':
            emails.append(parser.dataArray[i+1]+'@'+parser.dataArray[i+2])
            continue
        if parser.dataArray[i] == 'Login password':
            passwords.append(parser.dataArray[i+1])
            continue

    if ( userIds!=[] and names!=[] and emails!=[] and passwords!=[] ):
        database.insertTestUsers(userIds,names,emails,passwords)
        return 1
    else:
        return -1
    '''
    try:
        for i in range(len(names)):
            print names[i] + ' ' + userIds[i] + ' ' + emails[i] + ' ' + passwords[i] + ' '
    except:
        print 'for error in MyParser'
    '''

def parseOnline(data):
    buddies = []
    start = 0
    while True:
        match = re.search("fbid=", data[start:])
        if match is not None:
            start += match.end()
            matchBis = re.search("&",data[start:])
            if matchBis is not None:
                end = matchBis.end() + start
                buddies.append(str(data[start:end-1]))
                start = end
                end = 0
        else:
            break
    return buddies

def parseFriends(data):
    start = 0
    end = 0
    lines = []
    friends = []
    while True:
        match = re.search(r'href="/([a-zA-Z]*[0-9]*[\.]*)+(\?fref=fr_tab)',data[start:])
        if match is not None:
            lines.append(match.group())
            start += match.end()
        else:
            break

    for linea in lines:
        name = linea.split('/')[1].split('?')[0]
        friends.append(name)
    match = re.search("[a-zA-Z]+\?v=friends&mutual&startindex=[0-9]+",data)
    if match is not None:
        raw = match.group()
        next = raw.replace("&","&")
    else:
        match = re.search("[a-zA-Z]+/friends\?([a-zA-Z]+=[0-9]+)+(&)*([a-zA-Z]+=[0-9]+)*",data)
        if match is not None:
            raw = match.group()
            next = raw.replace("&","&")
        else:
            next = -1
    return friends,next

def parsePending():
    response = open("respuesta.html","r")
    struct = []
    aux = []
    while True:

        linea = response.readline()
        if not linea: break
        match = re.search('/ajax/reqs.php'+'(.+)',str(linea))
        if match is not None:
            struct.append(re.search('/ajax/reqs.php'+'(.+)',str(linea)).group())

    for lines in struct:
        start = 0


    mockReply.return_value = mockReplyInput
    result = jca._getAllowedJobTypes()
    assert result == expected

""" Test class for Job Cleaning Agent
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pytest
from mock import MagicMock

# DIRAC Components
from DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent import JobCleaningAgent
from DIRAC import gLogger

gLogger.setLevel("DEBUG")

# Mock Objects
mockReply = MagicMock()
mockAM = MagicMock()
mockNone = MagicMock()
mockNone.return_value = None
mockJMC = MagicMock()

@pytest.fixture
def jca(mocker):
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.__init__")
    mocker.patch(
        "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule._AgentModule__moduleProperties",
        side_effect=lambda x, y=None: y,
        create=True,
    )
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM)
    mocker.patch(
        "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.getDistinctJobAttributes", side_effect=mockReply
    )
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)

    jca = JobCleaningAgent()
    jca.log = gLogger
    jca.log.setLevel("DEBUG")
    jca._AgentModule__configDefaults = mockAM
    jca.initialize()

    return jca

@pytest.mark.parametrize(
    "mockReplyInput, expected",
    [
        ({"OK": True, "Value": ""}, {"OK": True, "Value": []}),
        ({"OK": False, "Message": ""}, {"OK": False, "Message": ""}),
    ],
)
def test__getAllowedJobTypes(jca, mockReplyInput, expected):
    """Testing JobCleaningAgent()._getAllowedJobTypes()"""

@pytest.mark.parametrize(
    "mockReplyInput, expected",
    [
        ({"OK": True, "Value": ""}, {"OK": True, "Value": None}),
        ({"OK": False, "Message": ""}, {"OK": False, "Message": ""}),
    ],
)
def test_removeJobsByStatus(jca, mockReplyInput, expected):
    """Testing JobCleaningAgent().removeDeletedJobs()"""

    mockReply.return_value = mockReplyInput
    result = jca.removeDeletedJobs()
    assert result == expected

@pytest.mark.parametrize(
    "conditions, mockReplyInput, expected",
    [
        ({"JobType": "", "Status": "Deleted"}, {"OK": True, "Value": ""}, {"OK": True, "Value": None}),
        ({"JobType": "", "Status": "Deleted"}, {"OK": False, "Message": ""}, {"OK": False, "Message": ""}),
        ({"JobType": [], "Status": "Deleted"}, {"OK": True, "Value": ""}, {"OK": True, "Value": None}),
        (
            {"JobType": ["some", "status"], "Status": ["Deleted", "Cancelled"]},
            {"OK": True, "Value": ""},
            {"OK": True, "Value": None},
        ),
    ],
)
def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
    """Testing JobCleaningAgent().deleteJobsByStatus()"""

    mockReply.return_value = mockReplyInput
    result = jca.deleteJobsByStatus(conditions)
    assert result == expected

@pytest.mark.parametrize(
    "inputs, params, expected",
    [
        ([], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
        (["a", "b"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
        (
            [],
            {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
            {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}},
        ),
        (
            [],
            {
                "OK": True,
                "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}, 2: {"OutputSandboxLFN": "/some/other/lfn/2.txt"}},
            },
            {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt", 2: "/some/other/lfn/2.txt"}}},
        ),
        (
            ["a", "b"],
            {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
            {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}},
        ),
        (["a", "b"], {"OK": False}, {"OK": False}),
    ],
)
def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
    """Testing JobCleaningAgent().deleteJobOversizedSandbox()"""

    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.__init__")
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB", return_value=mockNone)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)

    jobCleaningAgent = JobCleaningAgent()
    jobCleaningAgent.log = gLogger
    jobCleaningAgent.log.setLevel("DEBUG")
    jobCleaningAgent._AgentModule__configDefaults = mockAM
    jobCleaningAgent.initialize()

    mockJMC.getJobParameters.return_value = params

    result = jobCleaningAgent.deleteJobOversizedSandbox(inputs)

    assert result == expected


        self.id = id
        self.name = name
        self.class_def = class_def
        self.args = args

from abc import ABC
from typing import Dict, Any, Callable

class BaseContainer(ABC):
    """
        Base container class, for easier definition of containers. Ensures consistent format
        before being turned into a dataframe row.
    """

    def __init__(self, id: str, name: str, class_def: Callable, args: Dict[str, Any] = None):
        """
            class_def: The callable to be invoked
        """

    def get_dict(self, internal=False) -> Dict[str, Any]:
        d = [("ID", self.id), ("Name", self.name)]

        if internal:
            d += [
                ("Class", self.class_def),
                ("Args", self.args),
            ]

        return dict(d)


    offset = X.floor()
    scaled_remainder = (X - offset - 0.5) / tau
    rounding_component = (torch.tanh(scaled_remainder) + 1) / 2
    return offset + rounding_component

from __future__ import annotations

import torch
from torch import Tensor

def approximate_round(X: Tensor, tau: float = 1e-3) -> Tensor:
    r"""Diffentiable approximate rounding function.

    This method is a piecewise approximation of a rounding function where
    each piece is a hyperbolic tangent function.

    Args:
        X: The tensor to round to the nearest integer (element-wise).
        tau: A temperature hyperparameter.

    Returns:
        The approximately rounded input tensor.
    """


        if self.item(item_name=item_name, group_name=group_name):
            re = self.del_item(item_name=item_name, group_name=group_name)
            if isinstance(re, tuple):
                raise re[1]
        _cf_item = Config_Items(item_name=item_name, group_name=group_name, item_value=item_value)
        _cf_item.save()

# 该文件是白泽自动化管理系统的一部分,是白泽系统负责读取配置项的API接口

import logging
from baize.settings import *
from APP.APP_proxy.models import Config_Items

class Config(object):
    """ 配置项管理类 """
    def __init__(self):
        pass

    def add_item(self, item_name=None, group_name=None, item_value=None):
        """ 添加配置项,如果已经存在则进行更新 """

    def del_item(self, item_name=None, group_name=None):
        """ 删除配置项 """
        if not item_name:
            return False
        if not group_name:
            return False
        try:
            re = Config_Items.objects.filter(item_name=item_name, group_name=group_name).delete()
            return re
        except Exception, e:
            logger = logging.getLogger('log_file')
            logger.error(e)
            if LOG_SCREEN == 'ON':
                logger = logging.getLogger('log_screen')
                logger.error(e)
            re = False
            return re, e

    def item(self, item_name=None, group_name=None, default=None, type=None):
        """ 查询配置项 """
        if not item_name:
            if type is not None and (isinstance(default, str) or isinstance(default, unicode)):
                return eval(type)(default)
            else:
                return default
        if not group_name:
            if type is not None and (isinstance(default, str) or isinstance(default, unicode)):
                return eval(type)(default)
            else:
                return default
        try:
            re = Config_Items.objects.get(item_name=item_name, group_name=group_name).item_value
        except Exception, e:
            if default is not None:
                _config_items = Config_Items(item_name=item_name, group_name=group_name, item_value=default)
                _config_items.save()
            re = default
        if type is not None and (isinstance(re, str) or isinstance(re, unicode)):
            return eval(type)(re)
        else:
            return re


        # we really just need the hwnd, so setting "STATIC"
        # as predefined lpClass is just fine.
        hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0,
                                   None, None, None, None)
        try:
            yield hwnd
        finally:
            safeDestroyWindow(hwnd)

"""
This module implements clipboard handling on Windows using ctypes.
"""
import time
import contextlib
import ctypes
from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar
from .exceptions import PyperclipWindowsException

class CheckedCall(object):

    def __init__(self, f):
        super(CheckedCall, self).__setattr__("f", f)

    def __call__(self, *args):
        ret = self.f(*args)
        if not ret and get_errno():
            raise PyperclipWindowsException("Error calling " + self.f.__name__)
        return ret

    def __setattr__(self, key, value):
        setattr(self.f, key, value)

def init_windows_clipboard():
    from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND,
                                 HINSTANCE, HMENU, BOOL, UINT, HANDLE)

    windll = ctypes.windll

    safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA)
    safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT,
                                    INT, INT, HWND, HMENU, HINSTANCE, LPVOID]
    safeCreateWindowExA.restype = HWND

    safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow)
    safeDestroyWindow.argtypes = [HWND]
    safeDestroyWindow.restype = BOOL

    OpenClipboard = windll.user32.OpenClipboard
    OpenClipboard.argtypes = [HWND]
    OpenClipboard.restype = BOOL

    safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard)
    safeCloseClipboard.argtypes = []
    safeCloseClipboard.restype = BOOL

    safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard)
    safeEmptyClipboard.argtypes = []
    safeEmptyClipboard.restype = BOOL

    safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData)
    safeGetClipboardData.argtypes = [UINT]
    safeGetClipboardData.restype = HANDLE

    safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData)
    safeSetClipboardData.argtypes = [UINT, HANDLE]
    safeSetClipboardData.restype = HANDLE

    safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc)
    safeGlobalAlloc.argtypes = [UINT, c_size_t]
    safeGlobalAlloc.restype = HGLOBAL

    safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock)
    safeGlobalLock.argtypes = [HGLOBAL]
    safeGlobalLock.restype = LPVOID

    safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock)
    safeGlobalUnlock.argtypes = [HGLOBAL]
    safeGlobalUnlock.restype = BOOL

    GMEM_MOVEABLE = 0x0002
    CF_UNICODETEXT = 13

    @contextlib.contextmanager
    def window():
        """
        Context that provides a valid Windows hwnd.
        """

    @contextlib.contextmanager
    def clipboard(hwnd):
        """
        Context manager that opens the clipboard and prevents
        other applications from modifying the clipboard content.
        """
        # We may not get the clipboard handle immediately because
        # some other application is accessing it (?)
        # We try for at least 500ms to get the clipboard.
        t = time.time() + 0.5
        success = False
        while time.time() < t:
            success = OpenClipboard(hwnd)
            if success:
                break
            time.sleep(0.01)
        if not success:
            raise PyperclipWindowsException("Error calling OpenClipboard")

        try:
            yield
        finally:
            safeCloseClipboard()

    def copy_windows(text):
        # This function is heavily based on
        with window() as hwnd:
            # If an application calls OpenClipboard with hwnd set to NULL,
            # EmptyClipboard sets the clipboard owner to NULL;
            # this causes SetClipboardData to fail.
            # => We need a valid hwnd to copy something.
            with clipboard(hwnd):
                safeEmptyClipboard()

                if text:
                    # If the hMem parameter identifies a memory object,
                    # the object must have been allocated using the
                    # function with the GMEM_MOVEABLE flag.
                    count = len(text) + 1
                    handle = safeGlobalAlloc(GMEM_MOVEABLE,
                                             count * sizeof(c_wchar))
                    locked_handle = safeGlobalLock(handle)

                    ctypes.memmove(c_wchar_p(locked_handle),
                                   c_wchar_p(text), count * sizeof(c_wchar))

                    safeGlobalUnlock(handle)
                    safeSetClipboardData(CF_UNICODETEXT, handle)

    def paste_windows():
        with clipboard(None):
            handle = safeGetClipboardData(CF_UNICODETEXT)
            if not handle:
                # GetClipboardData may return NULL with errno == NO_ERROR
                # (Also, it may return a handle to an empty buffer,
                # but technically that's not empty)
                return ""
            return c_wchar_p(handle).value

    return copy_windows, paste_windows


while time.time() < t:
            success = OpenClipboard(hwnd)
            if success:
                break
            time.sleep(0.01)

"""
This module implements clipboard handling on Windows using ctypes.
"""
import time
import contextlib
import ctypes
from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar
from .exceptions import PyperclipWindowsException

class CheckedCall(object):

    def __init__(self, f):
        super(CheckedCall, self).__setattr__("f", f)

    def __call__(self, *args):
        ret = self.f(*args)
        if not ret and get_errno():
            raise PyperclipWindowsException("Error calling " + self.f.__name__)
        return ret

    def __setattr__(self, key, value):
        setattr(self.f, key, value)

def init_windows_clipboard():
    from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND,
                                 HINSTANCE, HMENU, BOOL, UINT, HANDLE)

    windll = ctypes.windll

    safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA)
    safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT,
                                    INT, INT, HWND, HMENU, HINSTANCE, LPVOID]
    safeCreateWindowExA.restype = HWND

    safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow)
    safeDestroyWindow.argtypes = [HWND]
    safeDestroyWindow.restype = BOOL

    OpenClipboard = windll.user32.OpenClipboard
    OpenClipboard.argtypes = [HWND]
    OpenClipboard.restype = BOOL

    safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard)
    safeCloseClipboard.argtypes = []
    safeCloseClipboard.restype = BOOL

    safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard)
    safeEmptyClipboard.argtypes = []
    safeEmptyClipboard.restype = BOOL

    safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData)
    safeGetClipboardData.argtypes = [UINT]
    safeGetClipboardData.restype = HANDLE

    safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData)
    safeSetClipboardData.argtypes = [UINT, HANDLE]
    safeSetClipboardData.restype = HANDLE

    safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc)
    safeGlobalAlloc.argtypes = [UINT, c_size_t]
    safeGlobalAlloc.restype = HGLOBAL

    safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock)
    safeGlobalLock.argtypes = [HGLOBAL]
    safeGlobalLock.restype = LPVOID

    safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock)
    safeGlobalUnlock.argtypes = [HGLOBAL]
    safeGlobalUnlock.restype = BOOL

    GMEM_MOVEABLE = 0x0002
    CF_UNICODETEXT = 13

    @contextlib.contextmanager
    def window():
        """
        Context that provides a valid Windows hwnd.
        """
        # we really just need the hwnd, so setting "STATIC"
        # as predefined lpClass is just fine.
        hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0,
                                   None, None, None, None)
        try:
            yield hwnd
        finally:
            safeDestroyWindow(hwnd)

    @contextlib.contextmanager
    def clipboard(hwnd):
        """
        Context manager that opens the clipboard and prevents
        other applications from modifying the clipboard content.
        """
        # We may not get the clipboard handle immediately because
        # some other application is accessing it (?)
        # We try for at least 500ms to get the clipboard.
        t = time.time() + 0.5
        success = False

        if not success:
            raise PyperclipWindowsException("Error calling OpenClipboard")

        try:
            yield
        finally:
            safeCloseClipboard()

    def copy_windows(text):
        # This function is heavily based on
        with window() as hwnd:
            # If an application calls OpenClipboard with hwnd set to NULL,
            # EmptyClipboard sets the clipboard owner to NULL;
            # this causes SetClipboardData to fail.
            # => We need a valid hwnd to copy something.
            with clipboard(hwnd):
                safeEmptyClipboard()

                if text:
                    # If the hMem parameter identifies a memory object,
                    # the object must have been allocated using the
                    # function with the GMEM_MOVEABLE flag.
                    count = len(text) + 1
                    handle = safeGlobalAlloc(GMEM_MOVEABLE,
                                             count * sizeof(c_wchar))
                    locked_handle = safeGlobalLock(handle)

                    ctypes.memmove(c_wchar_p(locked_handle),
                                   c_wchar_p(text), count * sizeof(c_wchar))

                    safeGlobalUnlock(handle)
                    safeSetClipboardData(CF_UNICODETEXT, handle)

    def paste_windows():
        with clipboard(None):
            handle = safeGetClipboardData(CF_UNICODETEXT)
            if not handle:
                # GetClipboardData may return NULL with errno == NO_ERROR
                # (Also, it may return a handle to an empty buffer,
                # but technically that's not empty)
                return ""
            return c_wchar_p(handle).value

    return copy_windows, paste_windows


while len(images) + len(added) < target_files_size:
                    offset = target_files_size - len(images) - len(added)
                    if offset >= len(images):
                        added.extend(images)
                    else:
                        images.extend(np.random.choice(images, offset, replace=False).tolist())

import numpy as np
import config
import json
import csv
import os

# 源文件路径
PATH_BASE_DIR = config.PATH_TRAIN_BASE
# PATH_BASE_DIR = config.PATH_VAL_BASE

# 保存文件路径
PATH_SAVE_DIR = os.path.join(PATH_BASE_DIR, 'classes')
# 是否按照分类名保存
SUB_DIR_WITH_NAME = False

PATH_IMAGES = os.path.join(PATH_BASE_DIR, 'scene_train_images_20170904')
PATH_JSON = os.path.join(PATH_BASE_DIR, 'scene_train_annotations_20170904.json')

PATH_CSV = os.path.join(PATH_BASE_DIR, 'scene_classes.csv')
PRINT = True
# 均值处理类不均衡问题
MEAN_HANDLE = False

def output(obj):
    if PRINT:
        if isinstance(obj, list) or isinstance(obj, tuple):
            for i in obj:
                print(i)
        else:
            print(obj)

def parse_labels():
    with open(PATH_CSV, encoding='utf-8') as f:
        return [line[1] for line in csv.reader(f)]

def parse_mapping():
    with open(PATH_JSON) as f:
        mapping = json.load(f)
        image2label = {item['image_id']: int(item['label_id']) for item in mapping}
        label2image = {}
        for image, label in image2label.items():
            if not label2image.__contains__(label):
                label2image[label] = []
            label2image[label].append(image)
        return image2label, label2image

if __name__ == '__main__':
    labels = parse_labels()
    output(labels[:5])

    image2label, label2image = parse_mapping()
    output(label2image[0][:5])

    for label, images in label2image.items():
        label_format = unicode(labels[label], 'utf-8') if SUB_DIR_WITH_NAME else ('%02d' % label)
        sub_dir = os.path.join(PATH_SAVE_DIR, label_format)
        if not os.path.exists(sub_dir):
            os.makedirs(sub_dir)
        if MEAN_HANDLE:
            target_files_size = len(image2label) // len(label2image)
            if len(images) > target_files_size:
                # 多了抽取
                images = np.random.choice(images, target_files_size, replace=False).tolist()
            elif len(images) < target_files_size:
                # 少了添加
                added = []

                images.extend(added)
        for image in images:
            with open(os.path.join(PATH_IMAGES, image), 'rb') as old:
                target_file = os.path.join(sub_dir, image)
                while os.path.exists(target_file):
                    target_file = target_file.replace('.', '_.')
                with open(target_file, 'wb') as new:
                    new.write(old.read())
                    output('Write finish % s' % image)
    output('Completed.')


    scheme = distutils_scheme(dist_name, user, home, root, isolated, prefix)
    return Scheme(
        platlib=scheme["platlib"],
        purelib=scheme["purelib"],
        headers=scheme["headers"],
        scripts=scheme["scripts"],
        data=scheme["data"],
    )

"""Locations where we look for configs, install stuff, etc"""

# The following comment should be removed at some point in the future.

import logging
import os
import sys
from distutils.cmd import Command as DistutilsCommand
from distutils.command.install import SCHEME_KEYS
from distutils.command.install import install as distutils_install_command
from distutils.sysconfig import get_python_lib
from typing import Dict, List, Optional, Tuple, Union, cast

from pip._internal.models.scheme import Scheme
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.virtualenv import running_under_virtualenv

from .base import get_major_minor_version

logger = logging.getLogger(__name__)

def distutils_scheme(
    dist_name: str,
    user: bool = False,
    home: str = None,
    root: str = None,
    isolated: bool = False,
    prefix: str = None,
    *,
    ignore_config_files: bool = False,
) -> Dict[str, str]:
    """
    Return a distutils install scheme
    """
    from distutils.dist import Distribution

    dist_args: Dict[str, Union[str, List[str]]] = {"name": dist_name}
    if isolated:
        dist_args["script_args"] = ["--no-user-cfg"]

    d = Distribution(dist_args)
    if not ignore_config_files:
        try:
            d.parse_config_files()
        except UnicodeDecodeError:
            paths = d.find_config_files()  # type: ignore
            logger.warning(
                "Ignore distutils configs in %s due to encoding errors.",
                ", ".join(os.path.basename(p) for p in paths),
            )
    obj: Optional[DistutilsCommand] = None
    obj = d.get_command_obj("install", create=True)
    assert obj is not None
    i = cast(distutils_install_command, obj)
    # NOTE: setting user or home has the side-effect of creating the home dir
    # or user base for installations during finalize_options()
    # ideally, we'd prefer a scheme class that has no side-effects.
    assert not (user and prefix), f"user={user} prefix={prefix}"
    assert not (home and prefix), f"home={home} prefix={prefix}"
    i.user = user or i.user
    if user or home:
        i.prefix = ""
    i.prefix = prefix or i.prefix
    i.home = home or i.home
    i.root = root or i.root
    i.finalize_options()

    scheme = {}
    for key in SCHEME_KEYS:
        scheme[key] = getattr(i, "install_" + key)

    # install_lib specified in setup.cfg should install *everything*
    # into there (i.e. it takes precedence over both purelib and
    # finalize_options(); we only want to override here if the user
    # has explicitly requested it hence going back to the config
    if "install_lib" in d.get_option_dict("install"):
        scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))

    if running_under_virtualenv():
        if home:
            prefix = home
        elif user:
            prefix = i.install_userbase  # type: ignore
        else:
            prefix = i.prefix
        scheme["headers"] = os.path.join(
            prefix,
            "include",
            "site",
            f"python{get_major_minor_version()}",
            dist_name,
        )

        if root is not None:
            path_no_drive = os.path.splitdrive(os.path.abspath(scheme["headers"]))[1]
            scheme["headers"] = os.path.join(root, path_no_drive[1:])

    return scheme

def get_scheme(
    dist_name: str,
    user: bool = False,
    home: Optional[str] = None,
    root: Optional[str] = None,
    isolated: bool = False,
    prefix: Optional[str] = None,
) -> Scheme:
    """
    Get the "scheme" corresponding to the input parameters. The distutils
    documentation provides the context for the available schemes:
    http://example.com

    :param dist_name: the name of the package to retrieve the scheme for, used
        in the headers scheme path
    :param user: indicates to use the "user" scheme
    :param home: indicates to use the "home" scheme and provides the base
        directory for the same
    :param root: root under which other directories are re-based
    :param isolated: equivalent to --no-user-cfg, i.e. do not consider
        ~/.pydistutils.cfg (posix) or ~/pydistutils.cfg (non-posix) for
        scheme paths
    :param prefix: indicates to use the "prefix" scheme and provides the
        base directory for the same
    """

def get_bin_prefix() -> str:
    # XXX: In old virtualenv versions, sys.prefix can contain '..' components,
    # so we need to call normpath to eliminate them.
    prefix = os.path.normpath(sys.prefix)
    if WINDOWS:
        bin_py = os.path.join(prefix, "Scripts")
        # buildout uses 'bin' on Windows too?
        if not os.path.exists(bin_py):
            bin_py = os.path.join(prefix, "bin")
        return bin_py
    # Forcing to use /usr/local/bin for standard macOS framework installs
    # Also log to ~/Library/Logs/ for use with the Console.app log viewer
    if sys.platform[:6] == "darwin" and prefix[:16] == "/System/Library/":
        return "/usr/local/bin"
    return os.path.join(prefix, "bin")

def get_purelib() -> str:
    return get_python_lib(plat_specific=False)

def get_platlib() -> str:
    return get_python_lib(plat_specific=True)

def get_prefixed_libs(prefix: str) -> Tuple[str, str]:
    return (
        get_python_lib(plat_specific=False, prefix=prefix),
        get_python_lib(plat_specific=True, prefix=prefix),
    )


    font = QtGui.QFont(family)
    # Check whether we got what we wanted using QFontInfo, since exactMatch()
    # is overly strict and returns false in too many cases.
    font_info = QtGui.QFontInfo(font)
    if fallback is not None and font_info.family() != family:
        font = QtGui.QFont(fallback)
    return font

""" Defines miscellaneous Qt-related helper classes and functions.
"""

import inspect

from qtpy import QtCore, QtGui

from traitlets import HasTraits, TraitType

# Metaclasses

MetaHasTraits = type(HasTraits)
MetaQObject = type(QtCore.QObject)

class MetaQObjectHasTraits(MetaQObject, MetaHasTraits):
    """ A metaclass that inherits from the metaclasses of HasTraits and QObject.

    Using this metaclass allows a class to inherit from both HasTraits and
    QObject. Using SuperQObject instead of QObject is highly recommended. See
    QtKernelManager for an example.
    """
    def __new__(mcls, name, bases, classdict):
        # FIXME: this duplicates the code from MetaHasTraits.
        # I don't think a super() call will help me here.
        for k,v in iter(classdict.items()):
            if isinstance(v, TraitType):
                v.name = k
            elif inspect.isclass(v):
                if issubclass(v, TraitType):
                    vinst = v()
                    vinst.name = k
                    classdict[k] = vinst
        cls = MetaQObject.__new__(mcls, name, bases, classdict)
        return cls

    def __init__(mcls, name, bases, classdict):
        # Note: super() did not work, so we explicitly call these.
        MetaQObject.__init__(mcls, name, bases, classdict)
        MetaHasTraits.__init__(mcls, name, bases, classdict)

# Classes

def superQ(QClass):
    """ Permits the use of super() in class hierarchies that contain Qt classes.

    Unlike QObject, SuperQObject does not accept a QObject parent. If it did,
    super could not be emulated properly (all other classes in the heierarchy
    would have to accept the parent argument--they don't, of course, because
    they don't inherit QObject.)

    This class is primarily useful for attaching signals to existing non-Qt
    classes. See QtKernelManagerMixin for an example.
    """
    class SuperQClass(QClass):

        def __new__(cls, *args, **kw):
            # We initialize QClass as early as possible. Without this, Qt complains
            inst = QClass.__new__(cls)
            QClass.__init__(inst)
            return inst

        def __init__(self, *args, **kw):
            # Emulate super by calling the next method in the MRO, if there is one.
            mro = self.__class__.mro()
            for qt_class in QClass.mro():
                mro.remove(qt_class)
            next_index = mro.index(SuperQClass) + 1
            if next_index < len(mro):
                mro[next_index].__init__(self, *args, **kw)

    return SuperQClass

SuperQObject = superQ(QtCore.QObject)

# Functions

def get_font(family, fallback=None):
    """Return a font of the requested family, using fallback as alternative.

    If a fallback is provided, it is used in case the requested family isn't
    found.  If no fallback is given, no alternative is chosen and Qt's internal
    algorithms may automatically choose a fallback font.

    Parameters
    ----------
    family : str
      A font name.
    fallback : str
      A font name.

    Returns
    -------
    font : QFont object
    """


    all_extensions = set()
    all_urls = kb.kb.get_all_known_urls()

    for url in all_urls:
        all_extensions.add(url.get_extension())

    return all_extensions

"""
kb_url_extensions.py

This file is part of w3af, http://example.com .

"""
import w3af.core.data.kb.knowledge_base as kb

def get_url_extensions_from_kb():
    """
    :return: A set with all the URL filename extensions that have been found
             during the scan. This is useful to reduce the number of HTTP
             requests which are sent during URL brute-forcing.

             For example, when a site only has aspx, js, css and png extensions
             a plugin might get that information using get_url_extensions_from_kb()
             and decide that it won't perform URL brute-forcing for php extensions.
    """


    hashfunc = hashlib.new(hash_algo)

    with path.open_r() as f_in:
        while True:
            data = f_in.read(bufsize)
            if not data:
                break
            hashfunc.update(data)

    return hashfunc.hexdigest()

"""
Functions for hashing files.
"""
import hashlib

def hash_file(path, hash_algo="sha3_256", bufsize=32768):
    """
    Get the hash value of a given file.

    :param path: Path of the file.
    :type path: .fslike.path.Path
    :param hash_algo: Hashing algorithm identifier.
    :type hash_algo: str
    :param bufsize: Buffer size for reading files.
    :type bufsize: int
    """


while True:
            data = f_in.read(bufsize)
            if not data:
                break
            hashfunc.update(data)

"""
Functions for hashing files.
"""
import hashlib

def hash_file(path, hash_algo="sha3_256", bufsize=32768):
    """
    Get the hash value of a given file.

    :param path: Path of the file.
    :type path: .fslike.path.Path
    :param hash_algo: Hashing algorithm identifier.
    :type hash_algo: str
    :param bufsize: Buffer size for reading files.
    :type bufsize: int
    """
    hashfunc = hashlib.new(hash_algo)

    with path.open_r() as f_in:


    if request.method == "POST":
        form = AuthTokenForm(request.POST)
        if form.is_valid():
            form.save(commit=False)
            form.instance.user = request.user
            form.instance.save()
            return HttpResponseRedirect(
                reverse("linaro_django_xmlrpc_tokens"))
    else:
        form = AuthTokenForm()
    return render(request, "linaro_django_xmlrpc/create_token.html",
                  {"form": form})

# Author: Zygmunt Krynicki <example@nnthink.com>
# This file is part of linaro-django-xmlrpc.

"""
XML-RPC views
"""

import base64

from django.contrib.auth.decorators import login_required
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse
from django.conf import settings
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect, render
from django.views.decorators.csrf import csrf_exempt

from linaro_django_xmlrpc.models import (
    AuthToken,
    CallContext,
    Dispatcher,
    SystemAPI,
)
from linaro_django_xmlrpc.forms import AuthTokenForm

from lava_server.bread_crumbs import (
    BreadCrumb,
    BreadCrumbTrail,
)
from lava_server.views import index as lava_index

@csrf_exempt
def handler(request, mapper, help_view):  # pylint: disable=too-many-return-statements
    """
    XML-RPC handler.

    If post data is defined, it assumes it's XML-RPC and tries to process as
    such. Empty POST request and GET requests assumes you're viewing from a
    browser and tells you about the service by redirecting you to a dedicated
    help page. For backwards compatibility the help view defaults to the
    'default_help' that shows what is registered in the global mapper. If you
    want to show help specific to your mapper you must specify help_view. It
    accepts whatever django.shortcuts.redirect() would.
    """
    if len(request.body):
        raw_data = request.body
        dispatcher = Dispatcher(mapper)

        auth_string = request.META.get('HTTP_AUTHORIZATION')

        if auth_string is not None:
            if ' ' not in auth_string:
                return HttpResponse("Invalid HTTP_AUTHORIZATION header", status=400)
            scheme, value = auth_string.split(" ", 1)
            if scheme != "Basic":
                return HttpResponse(
                    "Unsupported HTTP_AUTHORIZATION header, only Basic scheme is supported", status=400)
            try:
                decoded_value = base64.standard_b64decode(value).decode("utf-8")
            except (TypeError, UnicodeDecodeError):
                return HttpResponse("Corrupted HTTP_AUTHORIZATION header, bad base64 encoding", status=400)
            try:
                username, secret = decoded_value.split(":", 1)
            except ValueError:
                return HttpResponse("Corrupted HTTP_AUTHORIZATION header, no user:pass", status=400)
            user = AuthToken.get_user_for_secret(username, secret)
            if user is None:
                response = HttpResponse("Invalid token", status=401)
                response['WWW-Authenticate'] = 'Basic realm="XML-RPC Authentication token"'
                return response
        else:
            user = request.user
        result = dispatcher.marshalled_dispatch(raw_data, user, request)
        response = HttpResponse(content_type="application/xml")
        response.write(result)
        response['Content-length'] = str(len(response.content))
        return response
    else:
        return redirect(help_view)

@BreadCrumb("API help", parent=lava_index)
def help(request, mapper, template_name="linaro_django_xmlrpc/api.html"):  # pylint: disable=redefined-builtin
    context = CallContext(
        user=None, mapper=mapper, dispatcher=None)
    system = SystemAPI(context)
    if settings.HTTPS_XML_RPC:
        scheme = "https"
    else:
        scheme = request.META.get('REQUEST_SCHEME', "http")
    dashboard_methods = []
    scheduler_methods = []
    results_methods = []
    system_methods = []
    for method in system.listMethods():
        if 'dashboard' in method:
            dashboard_methods.append(method)
        elif 'scheduler' in method:
            scheduler_methods.append(method)
        elif 'results' in method:
            results_methods.append(method)
        else:
            system_methods.append(method)
    methods = {
        'dashboard': [
            {
                'name': method,
                'signature': system.methodSignature(method),
                'help': system.methodHelp(method)
            }
            for method in dashboard_methods
        ],
        'scheduler': [
            {
                'name': method,
                'signature': system.methodSignature(method),
                'help': system.methodHelp(method)
            }
            for method in scheduler_methods
        ],
        'results': [
            {
                'name': method,
                'signature': system.methodSignature(method),
                'help': system.methodHelp(method)
            }
            for method in results_methods
        ],
        'system': [
            {
                'name': method,
                'signature': system.methodSignature(method),
                'help': system.methodHelp(method)
            }
            for method in system_methods
        ]}
    domain = Site.objects.get_current().domain
    return render(request, template_name,
                  {'methods': methods,
                   'context_help': ['data-export'],
                   'bread_crumb_trail': BreadCrumbTrail.leading_to(help),
                   'site_scheme': scheme,
                   'site_domain': domain,
                   'site_url': "{scheme}://{domain}".format(scheme=scheme,
                                                            domain=domain)})

@BreadCrumb("API tokens", parent=lava_index)
@login_required
def tokens(request):
    """
    List of tokens for an authenticated user
    """
    token_list = AuthToken.objects.filter(user=request.user).order_by(
        "last_used_on")
    unused = AuthToken.objects.filter(user=request.user, last_used_on__isnull=True).count()
    return render(request, "linaro_django_xmlrpc/tokens.html",
                  {"token_list": token_list,
                   "unused": unused,
                   'bread_crumb_trail': BreadCrumbTrail.leading_to(tokens),
                   "context_help": ["first_steps"]})

@login_required
def create_token(request):
    """
    Create a token for the requesting user
    """

@login_required
def delete_token(request, object_id):
    token = get_object_or_404(AuthToken, pk=object_id, user=request.user)
    if request.method == 'POST':
        token.delete()
        return HttpResponseRedirect(
            reverse("linaro_django_xmlrpc_tokens"))
    return render(request, "linaro_django_xmlrpc/authtoken_confirm_delete.html",
                  {"token": token})

@login_required
def edit_token(request, object_id):
    token = get_object_or_404(AuthToken, pk=object_id, user=request.user)
    if request.method == "POST":
        form = AuthTokenForm(request.POST, instance=token)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(
                reverse("linaro_django_xmlrpc_tokens"))
    else:
        form = AuthTokenForm(instance=token)
    return render(request, "linaro_django_xmlrpc/edit_token.html",
                  {"token": token, "form": form})

@login_required
def delete_unused_tokens(request):
    token_list = AuthToken.objects.filter(user=request.user, last_used_on__isnull=True)
    if request.method == "POST":
        for token in token_list:
            token.delete()
        return HttpResponseRedirect(
            reverse("linaro_django_xmlrpc_tokens")
        )
    return render(request, "linaro_django_xmlrpc/tokens.html",
                  {"token_list": token_list, "context_help": ["lava-tool"]})