"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

"""Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b = self.hashbits
        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b


"""Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b = self.hashbits
        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b

"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

    def similarity(self, other_hash):


"""Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')

        b = self.hashbits
        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b


"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

    def similarity(self, other_hash):
        """Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')

        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b


if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b

"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

    def similarity(self, other_hash):
        """Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b = self.hashbits


"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

    def similarity(self, other_hash):
        """Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b


        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1
            for c in v:
                x = ((x*m) ^ ord(c)) & mask
            x ^= len(v)
            if x == -1:
                x = -2
            return x

"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"

    def similarity(self, other_hash):
        """Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b = self.hashbits
        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b


for c in v:
                x = ((x*m) ^ ord(c)) & mask

"""
Implementation of Charikar similarity hashes in Python.

Most useful for creating 'fingerprints' of documents or metadata
so you can quickly find duplicates or cluster items.

Part of python-hashes by sangelone. See README and LICENSE.
"""

from .hashtype import hashtype

class simhash(hashtype):
    def create_hash(self, tokens):
        """Calculates a Charikar simhash with appropriate bitlength.

        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns fingerprint so it can be used
        for temporary use without initializing a new object.
        Reference used: http://example.com
        """
        if type(tokens) == str:
            tokens = tokens.split()
        v = [0]*self.hashbits
        for t in [self._string_hash(x) for x in tokens]:
            bitmask = 0
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask:
                    v[i] += 1
                else:
                    v[i] -= 1

        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        self.hash = fingerprint
        return fingerprint

    def _string_hash(self, v):
        "A variable-length version of Python's builtin hash. Neat!"
        if v == "":
            return 0
        else:
            x = ord(v[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits-1

            x ^= len(v)
            if x == -1:
                x = -2
            return x

    def similarity(self, other_hash):
        """Calculate how similar this hash is from another simhash.
        Returns a float from 0.0 to 1.0 (linear distribution, inclusive)
        """
        if type(other_hash) != simhash:
            raise Exception('Hashes must be of same type to find similarity')
        b = self.hashbits
        if b != other_hash.hashbits:
            raise Exception('Hashes must be of equal size to find similarity')
        return float(b - self.hamming_distance(other_hash)) / b


import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):
    model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

    engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()
    if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist

run_pipeline(PIPE_CONFIG_PATH)

if __name__ == '__main__':
    test_engine()


import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):
    model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

    engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()
    if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist
def test_engine():

if __name__ == '__main__':
    test_engine()


model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):

    engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()
    if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist
def test_engine():
    run_pipeline(PIPE_CONFIG_PATH)

if __name__ == '__main__':
    test_engine()


engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()

import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):
    model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

    if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist
def test_engine():
    run_pipeline(PIPE_CONFIG_PATH)

if __name__ == '__main__':
    test_engine()


if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):
    model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

    engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist
def test_engine():
    run_pipeline(PIPE_CONFIG_PATH)

if __name__ == '__main__':
    test_engine()


import os.path as osp

import pytest
import torch

from colossalai import initialize
from colossalai.context import ParallelMode
from colossalai.core import global_context as gpc
from colossalai.engine import Engine
from colossalai.logging import get_global_dist_logger

NUM_BATCH = 128

BATCH_SIZE = 32
SEQ_LENGTH = 128
HIDDEN_SIZE = 512

DIR_PATH = osp.dirname(osp.realpath(__file__))
PIPE_CONFIG_PATH = osp.join(DIR_PATH, '../configs/pipeline_vanilla_resnet.py')

def run_pipeline(config):
    model, train_dataloader, test_dataloader, criterion, optimizer, schedule, lr_scheduler = initialize(config)
    logger = get_global_dist_logger()
    rank = torch.distributed.get_rank()
    engine = Engine(model=model,
                    train_dataloader=train_dataloader,
                    criterion=criterion,
                    optimizer=optimizer,
                    lr_scheduler=lr_scheduler,
                    schedule=schedule)

    engine.train()
    logger.info('lr = %g' % engine.get_lr())
    outputs, labels, loss = engine.step()
    if gpc.is_last_rank(ParallelMode.PIPELINE):
        logger.info('losses: {}'.format(rank, loss.item()))
    logger.info('lr = %g' % engine.get_lr())

    gpc.destroy()
    logger.info('Test engine pipeline finished')

@pytest.mark.skip("This test should be invoked using the test.sh provided")
@pytest.mark.dist
def test_engine():
    run_pipeline(PIPE_CONFI


"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod

data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}
        if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive mailbox serial data')

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}
        if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive mailbox serial data')

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}

"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):

        if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive mailbox serial data')

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]

"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}

        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive mailbox serial data')

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}
        if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive mailbox serial data')

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


should never receive mailbox serial data')

"""
Box Serializer (mailbox and parcelbox_
"""

from __future__ import absolute_import

import logging

from gateway.api.serializers.base import SerializerToolbox
from gateway.api.serializers import ApartmentSerializer
from gateway.dto import ParcelBoxDTO, MailBoxDTO
from ioc import Inject, INJECTED

if False:  # MYPY
    from typing import Any, Dict, Optional, List, Tuple

logger = logging.getLogger(__name__)

class MailboxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'open': dto_object.is_open,
                'apartment': None}
        if dto_object.apartment is not None:
            data['apartment'] = [ApartmentSerializer.serialize(dto_object.apartment)]
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api

class ParcelBoxSerializer(object):
    @staticmethod
    def serialize(dto_object, fields=None):
        data = {'id': dto_object.id,
                'label': str(dto_object.label),
                'height': dto_object.height,
                'width': dto_object.width,
                'size': dto_object.size.name,
                'available': dto_object.available,
                'open': dto_object.is_open}
        return SerializerToolbox.filter_fields(data, fields)

    @staticmethod
    def deserialize(api_data):
        raise NotImplementedError('Api should never receive parcelbox serial data')


'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

'''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id
#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names
    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


'''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id

'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):

#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names
    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names

'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):
    '''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id

    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


#print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):
    '''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id
#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):
    '''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id
#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names
    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all

##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):
    '''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id
#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]

    all_tf_names=cleaned_tf_names
    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


try:
        open(gin,'w').writelines([g+'\n' for g in gene_names])
        open(tin,'w').writelines([t+'\n' for t in all_tf_names])
    except:
        print "Error opening file:", sys.exc_info()[0]
        print "Check to make sure file exists at %s"%(closest_gene_output)
        raise

'''
From the original motif scanning, selects those events withing a particular window of a gene and outputs the
tgm, geneids and tf ids for final analysis
'''
__author__='sara jc gosline'
__email__='example@nnthink.com'

import re,os,sys
from optparse import OptionParser
from collections import defaultdict
import numpy as np

progdir=os.path.dirname(sys.argv[0])

def build_annotated_tgm(closest_gene_output,distance_to_tss,logistic_score_output,fasta_file,motif_ids,makeWindow=True,tgm_file='',do_pkl=True):
    '''
    Takes existing tgm, and maps to gene names and TF ids within a specific window
    '''
    from chipsequtil import Fasta
    ##get fasta file events, since these are columns in the logistic_score matrix
    seq_ids=Fasta.load(fasta_file,key_func=lambda x: x)

    ##need to get sequence mids in the order they are processed
    ##in the file, this is the index into the score_output file
    seq_mids=[] ##list of FASTA regions, in their appropriate order in the file
    filtered_events={}##gene name of closest gene to event within window
    for k in seq_ids.keys():
        vals=k.split(';')
        if len(vals)==1:
    	    vals=k.split()
        if ':' in vals[0]: #bed tools used
            chr,range=vals[0].split(':')
            low,high=range.split('-')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)
        elif 'random' not in vals[0]: #galaxy tools used
            genome,chr,low,high,strand=vals[0].split('_')
            mid=str(int(low)+((int(high)-int(low))/2))
            seq_mids.append(chr+':'+mid)

        if len(vals)==3:
            filtered_events[chr+':'+mid]=vals[2]
    print 'Found %d events, of which %d have gene names'%(len(seq_mids),len(filtered_events))
    ##this next section relies on xls
    ##filter events that are within distance from closest_gene_output to get gene mapping
    filtered_fc={}##FC of events within window, in case we want to use in the future

    event_indexes=[] ##

 #    ###open the closest_gene_output and determine
#         print "Error opening file:", sys.exc_info()[0]
#         print "Check to make sure file exists at %s"%(closest_gene_output)
#         if 'geneSymbol' in inds: #this is true if we used an xref file
#         else: #otherwise we just gene id
#         #position mapping is different

#         #print gene,mid
#             next

#         #check absolute distance if we're doing a window, or negative distance if we're looking upstream
# #            filtered_events[mid]=gene #(this was out of if clause, should it be there?) 1/2
#                 event_indexes.append(seq_mids.index(mid))##index into fasta file value/maps to array

#                 ##UPDATE: moved these to within if clause - so that unrelated scores are not included
#                 filtered_events[mid]=gene ##gene name of event
#                 filtered_fc[mid]=float(fc) ##fc value of event

  #  print 'Got '+str(len(filtered_events))+' per-gene events within '+distance_to_tss+' bp window out of '+str(len(cgo))

 #   print 'These map to '+str(len(event_indexes))+' regions in the FASTA file'

    ##get gene ids, or just use mid of sequence region
    gene_names=[t for t in set(filtered_events.values())]
    print gene_names[0:10]

    #get gene ids for all matrices list loaded in
    mi_files=motif_ids.split(',')
    if len(mi_files)>0:
        #open first motif name file that contains names for each element in TAMO file
        all_tf_names=[a.strip() for a in open(mi_files[0],'rU').readlines()]
    if len(mi_files)>1:
        #if we have additional files, check to see if if names already exist
        for i,f in enumerate(mi_files):
            if i==0:
                next
            try:
                #open file and read in extra ids
                newfs=[a.strip() for a in open(f,'rU').readlines()]
            except:
                print "Error opening file:", sys.exc_info()[0]
                print "Check to make sure file exists at %s"%(f)
                raise

            if len(newfs)==len(all_tf_names):
                #combine existing tf names with these with . delimiter....
                all_tf_names=['.'.join((a,b)) for a,b in zip(all_tf_names,newfs)]

    ##now go through and clean up TF names
    cleaned_tf_names=[]
    for i,a in enumerate(all_tf_names):
        tfn=set([b for b in a.split('.') if '$' not in b and b!=''])
        if(len(tfn)==0):
            tfn=a.split('.')
        cleaned_tf_names.append('.'.join(tfn))

    all_tf_names=cleaned_tf_names
    #print len(cleaned_tf_names)

    ##now actually map events to scores
    ##load motif matrix scanning output that maps matrices to regions
    print 'Loading complete motif score file...'
    event_scores=np.loadtxt(logistic_score_output)
    print '\t...Loaded!'

    #create new tgm matrix with approriate file name
    newmat=np.zeros((len(all_tf_names),len(gene_names)),dtype='float')##fill in gene length),dtype='float')
    if makeWindow:
        distance_to_tss=distance_to_tss+'_bpWindow'
    else:
        distance_to_tss=distance_to_tss+'_bpUpstream'

    if tgm_file=='':
        tgm_file=re.sub('.txt','_'+distance_to_tss+'.tgm',os.path.basename(logistic_score_output))
    if do_pkl:
        pkl_file=re.sub('.tgm','.pkl',tgm_file)
    else:
        pkl_file=''

    ##sort event indexes from seq_mids that are in the filtered_events file
    event_indexes.sort()

    #populate matrix with greatest score attributed to that gene/tf combo
    for ind,arr in enumerate(event_scores):
        ##name of matrix/motif
        mat=all_tf_names[ind]

        ##here we enumerate which sequences were mapped to a gene within the window
        for k,val in enumerate(seq_mids):#k in event_indexes:

            #here we want the event midpoint for the index

            #get score for that index
            score=arr[k]

            #now map it to closest gene for that midpoint
            cg=filtered_events[val]

            fc=1.0 ##update this if we want to normalize score by fold change
            score=float(score)*float(fc) ##this should do nothing sine fcgenerally =1

            curscore=newmat[all_tf_names.index(mat),gene_names.index(cg)]
            ##updated to include maximum score!!

            if np.abs(score)>np.abs(curscore):
                newmat[all_tf_names.index(mat),gene_names.index(cg)]=score
            #    ##updated to include maximum score!!

    ###save these intermediate files for debugging purposes
    np.savetxt(tgm_file,newmat)
    gin=re.sub('.tgm','_geneids.txt',tgm_file)
    tin=re.sub('.tgm','_tfids.txt',tgm_file)

    if pkl_file!='':
        zipcmd='python '+os.path.join(progdir,'zipTgms.py')+' '+tgm_file+' '+tin+' '+gin+' --pkl='+pkl_file
        print 'Compressing matrix file into pkl'
        print zipcmd
        os.system(zipcmd)
        return pkl_file
    else:
        return tgm_file

def main():
    '''
    main method
    '''
    usage='usage: %prog [options] motif_scanning_file closest_gene_output fasta_file'
    parser=OptionParser(usage=usage)
    parser.add_option('--distance-to-gene',dest='distance',type='string',default='10000',help='Max distance allowed between event and closest gene')
    parser.add_option('--motif-id-list',dest='motif_ids',type='string',default='',help='Comma-delimited list of files containing TF names that best map to each motif')

    parser.add_option('--utilpath',default=os.path.join(progdir,'../src/'),dest='addpath',help='Destination of chipsequtil library')
    parser.add_option('--outfile',default='',dest='outfile',help='Predefined output file, otherwise will create one automatically')
    parser.add_option('--noPkl',default=True,action='store_false',dest='do_pkl',help='Set this flag if outfile is not in pkl form')

    opts,args=parser.parse_args()

    if len(args)!=3:
        print usage
        exit('Not enough arguments')

    logistic_score_output,closest_gene_output,fasta_file=args

    ##do the path management
    sys.path.insert(0,opts.addpath)
#    print sys.path
#    print opts.do_pkl

    res=build_annotated_tgm(closest_gene_output,opts.distance,logistic_score_output,fasta_file,opts.motif_ids,tgm_file=opts.outfile,do_pkl=opts.do_pkl)

if __name__=='__main__':
    main()


import sys, os
import git
from PyQt5.QtWidgets import QApplication
import Init, GitNote

gitNoteHome = ""
gitNoteNoteHome = ""

userName = ""
password = ""
gitUrl = ""

gitExist = True

myGitNote = ""
updateStatus = False

os.environ['GIT_ASKPASS'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'askpass.py')
    os.environ['GIT_USERNAME'] = userName
    os.environ['GIT_PASSWORD'] = password

if __name__ == '__main__':
    app = QApplication(sys.argv)
    myInit = Init.Init()
    sys.exit(app.exec_())


os.environ['GIT_ASKPASS'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'askpass.py')
    os.environ['GIT_USERNAME'] = userName
    os.environ['GIT_PASSWORD'] = password

import sys, os
import git
from PyQt5.QtWidgets import QApplication
import Init, GitNote

gitNoteHome = ""
gitNoteNoteHome = ""

userName = ""
password = ""
gitUrl = ""

gitExist = True

myGitNote = ""
updateStatus = False

def setGitEnv():

if __name__ == '__main__':
    app = QApplication(sys.argv)
    myInit = Init.Init()
    sys.exit(app.exec_())


import sys, os
import git
from PyQt5.QtWidgets import QApplication
import Init, GitNote

gitNoteHome = ""
gitNoteNoteHome = ""

userName = ""
password = ""
gitUrl = ""

gitExist = True

myGitNote = ""
updateStatus = False

def setGitEnv():
    os.environ['GIT_ASKPASS'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'askpass.py')
    os.environ['GIT_USERNAME'] = userName
    os.environ['GIT_PA