while True:
        line = source.readline()
        # Check if we are out of input
        if line == '':
            return None
        # strip whitespace and see if this is then end of the parse
        line = line.strip()
        if line == '':
            break
        cur_text.append(line)

from pstree import *

ptb_tag_set = set(['S', 'SBAR', 'SBARQ', 'SINV', 'SQ', 'ADJP', 'ADVP', 'CONJP',
'FRAG', 'INTJ', 'LST', 'NAC', 'NP', 'NX', 'PP', 'PRN', 'PRT', 'QP', 'RRC',
'UCP', 'VP', 'WHADJP', 'WHADVP', 'WHNP', 'WHPP', 'X', 'NML'])

word_to_word_mapping = {
    '{': '-LCB-',
    '}': '-RCB-'
}
word_to_POS_mapping = {
    '--': ':',
    '-': ':',
    ';': ':',
    ':': ':',
    '-LRB-': '-LRB-',
    '-RRB-': '-RRB-',
    '-LCB-': '-LRB-',
    '-RCB-': '-RRB-',
    '{': '-LRB-',
    '}': '-RRB-',
}
bugfix_word_to_POS = {
    'Wa': 'NNP'
}
def ptb_cleaning(tree, in_place=True):
    '''Clean up some bugs/odd things in the PTB, and standardise punctuation.'''
    if not in_place:
        tree = tree.clone()
    for node in tree:
        # In a small number of cases multiple POS tags were assigned
        if '|' in node.label:
            if 'ADVP' in node.label:
                node.label = 'ADVP'
            else:
                node.label = node.label.split('|')[0]
        # Fix some issues with variation in output, and one error in the treebank
        # for a word with a punctuation POS
        if node.word in word_to_word_mapping:
            node.word = word_to_word_mapping[node.word]
        if node.word in word_to_POS_mapping:
            node.label = word_to_POS_mapping[node.word]
        if node.word in bugfix_word_to_POS:
            node.label = bugfix_word_to_POS[node.word]
    return tree

def remove_trivial_unaries(tree, in_place=True):
    '''Collapse A-over-A unary productions.

    >>> tree = tree_from_text("(ROOT (S (S (PP (PP (PP (IN By) (NP (CD 1997))))))))")
    >>> otree = remove_trivial_unaries(tree, False)
    >>> print otree
    (ROOT (S (PP (IN By) (NP (CD 1997)))))
    >>> print tree
    (ROOT (S (S (PP (PP (PP (IN By) (NP (CD 1997))))))))
    >>> remove_trivial_unaries(tree)
    (ROOT (S (PP (IN By) (NP (CD 1997)))))
    '''
    if in_place:
        if len(tree.subtrees) == 1 and tree.label == tree.subtrees[0].label:
            tree.subtrees = tree.subtrees[0].subtrees
            for subtree in tree.subtrees:
                subtree.parent = tree
            remove_trivial_unaries(tree, True)
        else:
            for subtree in tree.subtrees:
                remove_trivial_unaries(subtree, True)
    else:
        if len(tree.subtrees) == 1 and tree.label == tree.subtrees[0].label:
            return remove_trivial_unaries(tree.subtrees[0], False)
        subtrees = [remove_trivial_unaries(subtree, False) for subtree in tree.subtrees]
        tree = PSTree(tree.word, tree.label, tree.span, None, subtrees)
        for subtree in subtrees:
            subtree.parent = tree
    return tree

def remove_nodes(tree, filter_func, in_place=True, preserve_subtrees=False, init_call=True):
    if filter_func(tree) and not preserve_subtrees:
        return None
    subtrees = []
    for subtree in tree.subtrees:
        ans = remove_nodes(subtree, filter_func, in_place, preserve_subtrees, False)
        if ans is not None:
            if type(ans) == type([]):
                subtrees += ans
            else:
                subtrees.append(ans)
    if len(subtrees) == 0 and (not tree.is_terminal()):
        return None
    if filter_func(tree) and preserve_subtrees:
        return subtrees
    if in_place:
        tree.subtrees = subtrees
        for subtree in subtrees:
            subtree.parent = tree
    else:
        tree = PSTree(tree.word, tree.label, tree.span, None, subtrees)
    return tree

def remove_traces(tree, in_place=True):
    '''Adjust the tree to remove traces.

    >>> tree = tree_from_text("(ROOT (S (PP (IN By) (NP (CD 1997))) (, ,) (NP (NP (ADJP (RB almost) (DT all)) (VBG remaining) (NNS uses)) (PP (IN of) (NP (JJ cancer-causing) (NN asbestos)))) (VP (MD will) (VP (VB be) (VP (VBN outlawed) (NP (-NONE- *-6))))) (. .)))")
    >>> remove_traces(tree, False)
    (ROOT (S (PP (IN By) (NP (CD 1997))) (, ,) (NP (NP (ADJP (RB almost) (DT all)) (VBG remaining) (NNS uses)) (PP (IN of) (NP (JJ cancer-causing) (NN asbestos)))) (VP (MD will) (VP (VB be) (VP (VBN outlawed)))) (. .)))
    '''
    return remove_nodes(tree, PSTree.is_trace, in_place)

def split_label_type_and_function(label):
    parts = label.split('=')
    if len(label) > 0 and label[0] != '-':
        cur = parts
        parts = []
        for part in cur:
            parts += part.split('-')
    return parts

def remove_function_tags(tree, in_place=True):
    '''Adjust the tree to remove function tags on labels.

    >>> tree = tree_from_text("(ROOT (S (NP-SBJ (NNP Ms.) (NNP Haag)) (VP (VBZ plays) (NP (NNP Elianti))) (. .)))")
    >>> remove_function_tags(tree, False)
    (ROOT (S (NP (NNP Ms.) (NNP Haag)) (VP (VBZ plays) (NP (NNP Elianti))) (. .)))

    # don't remove brackets
    >>> tree = tree_from_text("(ROOT (S (NP-SBJ (`` ``) (NP-TTL (NNP Funny) (NNP Business)) ('' '') (PRN (-LRB- -LRB-) (NP (NNP Soho)) (, ,) (NP (CD 228) (NNS pages)) (, ,) (NP ($ $) (CD 17.95)) (-RRB- -RRB-)) (PP (IN by) (NP (NNP Gary) (NNP Katzenstein)))) (VP (VBZ is) (NP-PRD (NP (NN anything)) (PP (RB but)))) (. .)))")
    >>> remove_function_tags(tree)
    (ROOT (S (NP (`` ``) (NP (NNP Funny) (NNP Business)) ('' '') (PRN (-LRB- -LRB-) (NP (NNP Soho)) (, ,) (NP (CD 228) (NNS pages)) (, ,) (NP ($ $) (CD 17.95)) (-RRB- -RRB-)) (PP (IN by) (NP (NNP Gary) (NNP Katzenstein)))) (VP (VBZ is) (NP (NP (NN anything)) (PP (RB but)))) (. .)))
    '''
    label = split_label_type_and_function(tree.label)[0]
    if in_place:
        for subtree in tree.subtrees:
            remove_function_tags(subtree, True)
        tree.label = label
    else:
        subtrees = [remove_function_tags(subtree, False) for subtree in tree.subtrees]
        tree = PSTree(tree.word, label, tree.span, None, subtrees)
        for subtree in subtrees:
            subtree.parent = tree
    return tree

# Applies rules to strip out the parts of the tree that are not used in the
# standard evalb evaluation
def apply_collins_rules(tree, in_place=True):
    '''Adjust the tree to remove parts not evaluated by the standard evalb
    config.

    # cutting punctuation and -X parts of labels
    >>> tree = tree_from_text("(ROOT (S (NP-SBJ (NNP Ms.) (NNP Haag) ) (VP (VBZ plays) (NP (NNP Elianti) )) (. .) ))")
    >>> apply_collins_rules(tree)
    (ROOT (S (NP (NNP Ms.) (NNP Haag)) (VP (VBZ plays) (NP (NNP Elianti)))))
    >>> print tree.word_yield()
    Ms. Haag plays Elianti

    # cutting nulls
    >>> tree = tree_from_text("(ROOT (S (PP-TMP (IN By) (NP (CD 1997))) (, ,) (NP-SBJ-6 (NP (ADJP (RB almost) (DT all)) (VBG remaining) (NNS uses)) (PP (IN of) (NP (JJ cancer-causing) (NN asbestos)))) (VP (MD will) (VP (VB be) (VP (VBN outlawed) (NP (-NONE- *-6))))) (. .)))")
    >>> apply_collins_rules(tree)
    (ROOT (S (PP (IN By) (NP (CD 1997))) (NP (NP (ADJP (RB almost) (DT all)) (VBG remaining) (NNS uses)) (PP (IN of) (NP (JJ cancer-causing) (NN asbestos)))) (VP (MD will) (VP (VB be) (VP (VBN outlawed))))))

    # changing PRT to ADVP
    >>> tree = tree_from_text("(ROOT (S (NP-SBJ-41 (DT That) (NN fund)) (VP (VBD was) (VP (VBN put) (NP (-NONE- *-41)) (PRT (RP together)) (PP (IN by) (NP-LGS (NP (NNP Blackstone) (NNP Group)) (, ,) (NP (DT a) (NNP New) (NNP York) (NN investment) (NN bank)))))) (. .)))")
    >>> apply_collins_rules(tree)
    (ROOT (S (NP (DT That) (NN fund)) (VP (VBD was) (VP (VBN put) (ADVP (RP together)) (PP (IN by) (NP (NP (NNP Blackstone) (NNP Group)) (NP (DT a) (NNP New) (NNP York) (NN investment) (NN bank))))))))

    # not removing brackets
    >>> tree = tree_from_text("(ROOT (S (NP-SBJ (`` ``) (NP-TTL (NNP Funny) (NNP Business)) ('' '') (PRN (-LRB- -LRB-) (NP (NNP Soho)) (, ,) (NP (CD 228) (NNS pages)) (, ,) (NP ($ $) (CD 17.95) (-NONE- *U*)) (-RRB- -RRB-)) (PP (IN by) (NP (NNP Gary) (NNP Katzenstein)))) (VP (VBZ is) (NP-PRD (NP (NN anything)) (PP (RB but) (NP (-NONE- *?*))))) (. .)))")
    >>> apply_collins_rules(tree)
    (ROOT (S (NP (NP (NNP Funny) (NNP Business)) (PRN (-LRB- -LRB-) (NP (NNP Soho)) (NP (CD 228) (NNS pages)) (NP ($ $) (CD 17.95)) (-RRB- -RRB-)) (PP (IN by) (NP (NNP Gary) (NNP Katzenstein)))) (VP (VBZ is) (NP (NP (NN anything)) (PP (RB but))))))
    '''
    tree = tree if in_place else tree.clone()
    remove_traces(tree, True)
    remove_function_tags(tree, True)
    ptb_cleaning(tree, True)

    # Remove Puncturation
    labels_to_ignore = ["-NONE-", ",", ":", "``", "''", "."]
    remove_nodes(tree, lambda(t): t.label in labels_to_ignore, True)

    # Set all PRTs to be ADVPs
    POS_to_convert = {'PRT': 'ADVP'}
    for node in tree:
        if node.label in POS_to_convert:
            node.label = POS_to_convert[node.label]

    tree.calculate_spans()
    return tree

def homogenise_tree(tree, tag_set=ptb_tag_set):
    '''Change the top of the tree to be of a consistent form.

    >>> tree = tree_from_text("( (S (NP (NNP Example))))", True)
    >>> homogenise_tree(tree)
    (ROOT (S (NP (NNP Example))))
    >>> tree = tree_from_text("( (ROOT (S (NP (NNP Example))) ) )", True)
    >>> homogenise_tree(tree)
    (ROOT (S (NP (NNP Example))))
    >>> tree = tree_from_text("(S1 (S (NP (NNP Example))))")
    >>> homogenise_tree(tree)
    (ROOT (S (NP (NNP Example))))
    '''
    orig = tree
    tree = tree.root()
    if tree.label != 'ROOT':
        while split_label_type_and_function(tree.label)[0] not in tag_set:
            if len(tree.subtrees) > 1:
                break
            elif tree.is_terminal():
                raise Exception("Tree has no labels in the tag set\n%s" % orig.__repr__())
            tree = tree.subtrees[0]
        if split_label_type_and_function(tree.label)[0] not in tag_set:
            tree.label = 'ROOT'
        else:
            root = PSTree(None, 'ROOT', tree.span, None, [])
            root.subtrees.append(tree)
            tree.parent = root
            tree = root
    return tree

def ptb_read_tree(source, return_empty=False, allow_empty_labels=False, allow_empty_words=False, blank_line_coverage=False):
    '''Read a single tree from the given PTB file.

    The function reads a character at a time, stopping as soon as a tree can be
    constructed, so multiple trees on a sinlge line are manageable.

    >>> from StringIO import StringIO
    >>> file_text = """(ROOT (S
    ...   (NP-SBJ (NNP Scotty) )
    ...   (VP (VBD did) (RB not)
    ...     (VP (VB go)
    ...       (ADVP (RB back) )
    ...       (PP (TO to)
    ...         (NP (NN school) ))))
    ...   (. .) ))"""
    >>> in_file = StringIO(file_text)
    >>> ptb_read_tree(in_file)
    (ROOT (S (NP-SBJ (NNP Scotty)) (VP (VBD did) (RB not) (VP (VB go) (ADVP (RB back)) (PP (TO to) (NP (NN school))))) (. .)))'''
    cur_text = ''
    depth = 0
    while True:
        char = source.read(1)
        if char == '':
            return None
        if char == '\n' and cur_text == ' ' and blank_line_coverage:
            return "Empty"
        if char in '\n\t':
            char = ' '
        cur_text += char
        if char == '(':
            depth += 1
        elif char == ')':
            depth -= 1
        if depth == 0:
            if '()' in cur_text:
                if return_empty:
                    return "Empty"
                cur_text = ''
                continue
            if '(' in cur_text:
                break

    tree = tree_from_text(cur_text, allow_empty_labels, allow_empty_words)
    ptb_cleaning(tree)
    return tree

def conll_read_tree(source, return_empty=False, allow_empty_labels=False, allow_empty_words=False, blank_line_coverage=False):
    '''Read a single tree from the given CoNLL Shared Task OntoNotes data file.

    >>> from StringIO import StringIO
    >>> file_text = """#begin document (nw/wsj/00/wsj_0020)
    ... nw/wsj/00/wsj_0020          0          0       They        PRP (TOP_(S_(NP_*)          -          -          -          -          * (ARG1*)          *        (0)
    ... nw/wsj/00/wsj_0020          0          1       will         MD      (VP_*          -          -          -          -          * (ARGM-M OD*)          *          -
    ... nw/wsj/00/wsj_0020          0          2     remain         VB      (VP_*     remain         01          1          -          *       ( V*)          *          -
    ... nw/wsj/00/wsj_0020          0          3         on         IN      (PP_*          -          -          -          -          *     (AR G3*          *          -
    ... nw/wsj/00/wsj_0020          0          4          a         DT  (NP_(NP_*          -          -          -          -          * *     (ARG2*          -
    ... nw/wsj/00/wsj_0020          0          5      lower        JJR     (NML_*          -          -          -          -          * *          *          -
    ... nw/wsj/00/wsj_0020          0          6          -       HYPH          *          -          -          -          -          * *          *          -
    ... nw/wsj/00/wsj_0020          0          7   priority         NN         *)          -          -          -          -          * *          *          -
    ... nw/wsj/00/wsj_0020          0          8       list         NN         *)          -          -          1          -          * *         *)          -
    ... nw/wsj/00/wsj_0020          0          9       that        WDT (SBAR_(WHNP_*)          -          -          -          -          * *          *          -
    ... nw/wsj/00/wsj_0020          0         10   includes        VBZ   (S_(VP_*          -          -          1          -          * *       (V*)          -
    ... nw/wsj/00/wsj_0020          0         11         17         CD      (NP_*          -          -          -          - (CARDINAL) *     (ARG1*        (10
    ... nw/wsj/00/wsj_0020          0         12      other         JJ          *          -          -          -          -          * *          *          -
    ... nw/wsj/00/wsj_0020          0         13  countries        NNS  *))))))))          -          -          3          -          * *)         *)        10)
    ... nw/wsj/00/wsj_0020          0         14          .          .        *))          -          -          -          -          * *          *          -
    ...
    ... """
    >>> in_file = StringIO(file_text)
    >>> tree = conll_read_tree(in_file)
    >>> print tree
    (TOP (S (NP (PRP They)) (VP (MD will) (VP (VB remain) (PP (IN on) (NP (NP (DT a) (NML (JJR lower) (HYPH -) (NN priority)) (NN list)) (SBAR (WHNP (WDT that)) (S (VP (VBZ includes) (NP (CD 17) (JJ other) (NNS countries))))))))) (. .)))'''
    cur_text = []

    text = ''
    for line in cur_text:
        if len(line) == 0 or line[0] == '#':
            continue
        line = line.split()
        word = line[3]
        pos = line[4]
        tree = line[5]
        tree = tree.split('*')
        text += '%s(%s %s)%s' % (tree[0], pos, word, tree[1])
    return tree_from_text(text)

def generate_trees(source, tree_reader=ptb_read_tree, max_sents=-1, return_empty=False, allow_empty_labels=False, allow_empty_words=False):
    '''Read trees from the given file (opening the file if only a string is given).

    >>> from StringIO import StringIO
    >>> file_text = """(ROOT (S
    ...   (NP-SBJ (NNP Scotty) )
    ...   (VP (VBD did) (RB not)
    ...     (VP (VB go)
    ...       (ADVP (RB back) )
    ...       (PP (TO to)
    ...         (NP (NN school) ))))
    ...   (. .) ))
    ...
    ... (ROOT (S
    ...         (NP-SBJ (DT The) (NN bandit) )
    ...         (VP (VBZ laughs)
    ...             (PP (IN in)
    ...                 (NP (PRP$ his) (NN face) )))
    ...         (. .) ))"""
    >>> in_file = StringIO(file_text)
    >>> for tree in generate_trees(in_file):
    ...   print tree
    (ROOT (S (NP-SBJ (NNP Scotty)) (VP (VBD did) (RB not) (VP (VB go) (ADVP (RB back)) (PP (TO to) (NP (NN school))))) (. .)))
    (ROOT (S (NP-SBJ (DT The) (NN bandit)) (VP (VBZ laughs) (PP (IN in) (NP (PRP$ his) (NN face)))) (. .)))'''
    if type(source) == type(''):
        source = open(source)
    count = 0
    while True:
        tree = tree_reader(source, return_empty, allow_empty_labels, allow_empty_words)
        if tree == "Empty":
            yield None
            continue
        if tree is None:
            return
        yield tree
        count += 1
        if count >= max_sents > 0:
            return

def read_trees(source, tree_reader=ptb_read_tree, max_sents=-1, return_empty=False):
    return [tree for tree in generate_trees(source, tree_reader, max_sents, return_empty)]

if __name__ == '__main__':
    print "Running doctest"
    import doctest
    doctest.testmod()


def test_json_parsing(self, invoke, subcmd):

# All Rights Reserved.

# pylint: disable=redefined-outer-name

import pytest
import simplejson as json
from boltons.iterutils import same

from ..pool import DEFAULT_MANAGER_IDS
from .test_cli import CLISubCommandTests, CLITableTests

@pytest.fixture
def subcmd():
    return "installed"

class TestInstalled(CLISubCommandTests, CLITableTests):
    @pytest.mark.parametrize("mid", DEFAULT_MANAGER_IDS)
    def test_single_manager(self, invoke, subcmd, mid):
        result = invoke("--manager", mid, subcmd)
        assert result.exit_code == 0
        self.check_manager_selection(result, {mid})

result = invoke("--output-format", "json", subcmd)
        assert result.exit_code == 0
        data = json.loads(result.stdout)

        assert set(data).issubset(DEFAULT_MANAGER_IDS)

        for manager_id, info in data.items():
            assert isinstance(manager_id, str)
            assert isinstance(info, dict)

            assert set(info) == {"errors", "id", "name", "packages"}

            assert isinstance(info["errors"], list)
            if info["errors"]:
                assert same(map(type, info["errors"]), str)
            assert isinstance(info["id"], str)
            assert isinstance(info["name"], str)
            assert isinstance(info["packages"], list)

            assert info["id"] == manager_id

            for pkg in info["packages"]:
                assert isinstance(pkg, dict)

                assert set(pkg) == {"id", "installed_version", "name"}

                assert isinstance(pkg["id"], str)
                assert isinstance(pkg["installed_version"], str)
                assert isinstance(pkg["name"], str)


# All Rights Reserved.

# pylint: disable=redefined-outer-name

import pytest
import simplejson as json
from boltons.iterutils import same

from ..pool import DEFAULT_MANAGER_IDS
from .test_cli import CLISubCommandTests, CLITableTests

@pytest.fixture
def subcmd():

class TestInstalled(CLISubCommandTests, CLITableTests):
    @pytest.mark.parametrize("mid", DEFAULT_MANAGER_IDS)
    def test_single_manager(self, invoke, subcmd, mid):
        result = invoke("--manager", mid, subcmd)
        assert result.exit_code == 0
        self.check_manager_selection(result, {mid})

    def test_json_parsing(self, invoke, subcmd):
        result = invoke("--output-format", "json", subcmd)
        assert result.exit_code == 0
        data = json.loads(result.stdout)

        assert set(data).issubset(DEFAULT_MANAGER_IDS)

        for manager_id, info in data.items():
            assert isinstance(manager_id, str)
            assert isinstance(info, dict)

            assert set(info) == {"errors", "id", "name", "packages"}

            assert isinstance(info["errors"], list)
            if info["errors"]:
                assert same(map(type, info["errors"]), str)
            assert isinstance(info["id"], str)
            assert isinstance(info["name"], str)
            assert isinstance(info["packages"], list)

            assert info["id"] == manager_id

            for pkg in info["packages"]:
                assert isinstance(pkg, dict)

                assert set(pkg) == {"id", "installed_version", "name"}

                assert isinstance(pkg["id"], str)
                assert isinstance(pkg["installed_version"], str)
                assert isinstance(pkg["name"], str)


# All Rights Reserved.

# pylint: disable=redefined-outer-name

import pytest
import simplejson as json
from boltons.iterutils import same

from ..pool import DEFAULT_MANAGER_IDS
from .test_cli import CLISubCommandTests, CLITableTests

@pytest.fixture
def subcmd():
    return "installed"

class TestInstalled(CLISubCommandTests, CLITableTests):
    @pytest.mark.parametrize("mid", DEFAULT_MANAGER_IDS)
    def test_single_manager(self, invoke, subcmd, mid):
        result = invoke("--manager", mid, subcmd)
        assert result.exit_code == 0
        self.check_manager_selection(result, {mid})

    def test_json_parsing(self, invoke, subcmd):
        result = invoke("--

        assert result.exit_code == 0
        data = json.loads(result.stdout)

        assert set(data).issubset(DEFAULT_MANAGER_IDS)

        for manager_id, info in data.items():
            assert isinstance(manager_id, str)
            assert isinstance(info, dict)

            assert set(info) == {"errors", "id", "name", "packages"}

            assert isinstance(info["errors"], list)
            if info["errors"]:
                assert same(map(type, info["errors"]), str)
            assert isinstance(info["id"], str)
            assert isinstance(info["name"], str)
            assert isinstance(info["packages"], list)

            assert info["id"] == manager_id

            for pkg in info["packages"]:
                assert isinstance(pkg, dict)

                assert set(pkg) == {"id", "installed_version", "name"}

                assert isinstance(pkg["id"], str)
                assert isinstance(pkg["installed_version"], str)
                assert isinstance(pkg["name"], str)


for pkg in info["packages"]:
                assert isinstance(pkg, dict)

                assert set(pkg) == {"id", "installed_version", "name"}

                assert isinstance(pkg["id"], str)
                assert isinstance(pkg["installed_version"], str)
                assert isinstance(pkg["name"], str)

# All Rights Reserved.

# pylint: disable=redefined-outer-name

import pytest
import simplejson as json
from boltons.iterutils import same

from ..pool import DEFAULT_MANAGER_IDS
from .test_cli import CLISubCommandTests, CLITableTests

@pytest.fixture
def subcmd():
    return "installed"

class TestInstalled(CLISubCommandTests, CLITableTests):
    @pytest.mark.parametrize("mid", DEFAULT_MANAGER_IDS)
    def test_single_manager(self, invoke, subcmd, mid):
        result = invoke("--manager", mid, subcmd)
        assert result.exit_code == 0
        self.check_manager_selection(result, {mid})

    def test_json_parsing(self, invoke, subcmd):
        result = invoke("--output-format", "json", subcmd)
        assert result.exit_code == 0
        data = json.loads(result.stdout)

        assert set(data).issubset(DEFAULT_MANAGER_IDS)

        for manager_id, info in data.items():
            assert isinstance(manager_id, str)
            assert isinstance(info, dict)

            assert set(info) == {"errors", "id", "name", "packages"}

            assert isinstance(info["errors"], list)
            if info["errors"]:
                assert same(map(type, info["errors"]), str)
            assert isinstance(info["id"], str)
            assert isinstance(info["name"], str)
            assert isinstance(info["packages"], list)

            assert info["id"] == manager_id


from aliyunsdkcore.request import RpcRequest
from aliyunsdkhbase.endpoint import endpoint_data

class ListInstanceServiceConfigurationsRequest(RpcRequest):

	def __init__(self):
		RpcRequest.__init__(self, 'HBase', '2019-01-01', 'ListInstanceServiceConfigurations','hbase')
		self.set_method('POST')
		if hasattr(self, "endpoint_map"):
			setattr(self, "endpoint_map", endpoint_data.getEndpointMap())
		if hasattr(self, "endpoint_regional"):
			setattr(self, "endpoint_regional", endpoint_data.getEndpointRegional())

return self.get_query_params().get('ClusterId')

	def set_ClusterId(self,ClusterId):
		self.add_query_param('ClusterId',ClusterId)

	def get_PageNumber(self):
		return self.get_query_params().get('PageNumber')

	def set_PageNumber(self,PageNumber):
		self.add_query_param('PageNumber',PageNumber)

	def get_PageSize(self):
		return self.get_query_params().get('PageSize')

	def set_PageSize(self,PageSize):
		self.add_query_param('PageSize',PageSize)


from aliyunsdkcore.request import RpcRequest
from aliyunsdkhbase.endpoint import endpoint_data

class ListInstanceServiceConfigurationsRequest(RpcRequest):

	def __init__(self):
		RpcRequest.__init__(self, 'HBase', '2019-01-01', 'ListInstanceServiceConfigurations','hbase')
		self.set_method('POST')
		if hasattr(self, "endpoint_map"):
			setattr(self, "endpoint_map", endpoint_data.getEndpointMap())
		if hasattr(self, "endpoint_regional"):
			setattr(self, "endpoint_regional", endpoint_data.getEndpointRegional())

	def get_ClusterId(self):
		return self.get_query_params().get('ClusterId')

	def set_ClusterId(self,ClusterId):
		self.add_query_param('ClusterId',ClusterId)

	def get_PageNumber(self):

	def set_PageNumber(self,PageNumber):
		self.add_query_param('PageNumber',PageNumber)

	def get_PageSize(self):
		return self.get_query_params().get('PageSize')

	def set_PageSize(self,PageSize):
		self.add_query_param('PageSize',PageSize)


from aliyunsdkcore.request import RpcRequest
from aliyunsdkhbase.endpoint import endpoint_data

class ListInstanceServiceConfigurationsRequest(RpcRequest):

	def __init__(self):
		RpcRequest.__init__(self, 'HBase', '2019-01-01', 'ListInstanceServiceConfigurations','hbase')
		self.set_method('POS

		if hasattr(self, "endpoint_map"):
			setattr(self, "endpoint_map", endpoint_data.getEndpointMap())
		if hasattr(self, "endpoint_regional"):
			setattr(self, "endpoint_regional", endpoint_data.getEndpointRegional())

	def get_ClusterId(self):
		return self.get_query_params().get('ClusterId')

	def set_ClusterId(self,ClusterId):
		self.add_query_param('ClusterId',ClusterId)

	def get_PageNumber(self):
		return self.get_query_params().get('PageNumber')

	def set_PageNumber(self,PageNumber):
		self.add_query_param('PageNumber',PageNumber)

	def get_PageSize(self):
		return self.get_query_params().get('PageSize')

	def set_PageSize(self,PageSize):
		self.add_query_param('PageSize',PageSize)


"""Auto posting to board by type and category

Revision ID: 576b32d25ea2
Revises: 57da2d78c9ca
Create Date: 2016-05-02 13:49:16.797507

"""

# revision identifiers, used by Alembic.
revision = '576b32d25ea2'
down_revision = '57da2d78c9ca'

from alembic import op
import sqlalchemy as sa

op.create_table(
        'board_auto_jobcategory',
        sa.Column('jobcategory_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobcategory_id'], ['jobcategory.id']),
        sa.PrimaryKeyConstraint('jobcategory_id', 'board_id'),
    )
    op.create_table(
        'board_auto_jobtype',
        sa.Column('jobtype_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobtype_id'], ['jobtype.id']),
        sa.PrimaryKeyConstraint('jobtype_id', 'board_id'),
    )
    op.add_column(
        'board', sa.Column('auto_all', sa.Boolean(), nullable=False, server_default='0')
    )
    op.alter_column('board', 'auto_all', server_default=None)

def downgrade():
    op.drop_column('board', 'auto_all')
    op.drop_table('board_auto_jobtype')
    op.drop_table('board_auto_jobcategory')


op.drop_column('board', 'auto_all')
    op.drop_table('board_auto_jobtype')
    op.drop_table('board_auto_jobcategory')

"""Auto posting to board by type and category

Revision ID: 576b32d25ea2
Revises: 57da2d78c9ca
Create Date: 2016-05-02 13:49:16.797507

"""

# revision identifiers, used by Alembic.
revision = '576b32d25ea2'
down_revision = '57da2d78c9ca'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.create_table(
        'board_auto_jobcategory',
        sa.Column('jobcategory_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobcategory_id'], ['jobcategory.id']),
        sa.PrimaryKeyConstraint('jobcategory_id', 'board_id'),
    )
    op.create_table(
        'board_auto_jobtype',
        sa.Column('jobtype_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobtype_id'], ['jobtype.id']),
        sa.PrimaryKeyConstraint('jobtype_id', 'board_id'),
    )
    op.add_column(
        'board', sa.Column('auto_all', sa.Boolean(), nullable=False, server_default='0')
    )
    op.alter_column('board', 'auto_all', server_default=None)

def downgrade():


"""Auto posting to board by type and category

Revision ID: 576b32d25ea2
Revises: 57da2d78c9ca
Create Date: 2016-05-02 13:49:16.797507

"""

# revision identifiers, used by Alembic.
revision = '576b32d25ea2'
down_revision = '57da2d78c9ca'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.create_table(
        'board_auto_jobcategory',
        sa.Column('jobcategory_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobcategory_id'], ['jobcategory.id']),
        sa.PrimaryKeyConstraint('jobcategory_id', 'board_id'),
    )
    op.create_table(
        'board_auto_jobtype',
        sa.Column('jobtype_id', sa.Integer(), nullable=False),
        sa.Column('board_id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.ForeignKeyConstraint(['board_id'], ['board.id']),
        sa.ForeignKeyConstraint(['jobtype_id'], ['jobtype.id']),
        sa.PrimaryKeyConstraint('jobtype_id', 'board_id'),
    )
    op.add_column(
        'board', sa.Column('auto_all', sa.Boolean(), nullable=False, server_default='0')
    )
    op.alter_column('board', 'auto_all', serv

def downgrade():
    op.drop_column('board', 'auto_all')
    op.drop_table('board_auto_jobtype')
    op.drop_table('board_auto_jobcategory')


def _save_loop_state(epoch, logs, save_path, save_callbacks):

"""
Self-contained training loop for the project.
"""

import logging
import os
import pickle
from collections import defaultdict
from functools import partial
import gin
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow.keras.metrics import Metric as keras_metric

from src.callbacks import ModelCheckpoint, LambdaCallback, History, DumpTensorboardSummaries, BaseLogger, Heartbeat, \
    MetaSaver, MeasureTime
from src.utils import save_model, restore_model_and_optimizer, restore_model, configure_neptune_exp

logger = logging.getLogger(__name__)

def _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor, save_freq, custom_callbacks,
                                 use_tb, save_history_every_k_examples):
    callbacks = []
    callbacks.append(MetaSaver())
    callbacks.append(BaseLogger())
    callbacks.append(MeasureTime())
    callbacks.append(Heartbeat())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_append_to_history_csv, H=H)))
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_history_csv, save_path=save_path, H=H)))
    callbacks.append(History(save_every_k_examples=save_history_every_k_examples))

    if len(checkpoint_monitor):
        callbacks.append(ModelCheckpoint(monitor=checkpoint_monitor,
                                         save_best_only=True,
                                         mode='max',
                                         filepath=os.path.join(save_path, "model_best_val")))
    if save_freq > 0:
        def save_weights_fnc(epoch, logs):
            if epoch % save_freq == 0:
                logger.info("Saving model from epoch " + str(epoch))
                save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))

        callbacks.append(LambdaCallback(on_epoch_end=save_weights_fnc))

    if use_tb:
        callbacks.append(DumpTensorboardSummaries())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_loop_state, save_callbacks=custom_callbacks,
                                                         save_path=save_path)))
    return callbacks

loop_state = {"epochs_done": epoch, "callbacks": save_callbacks}
    pickle.dump(loop_state, open(os.path.join(save_path, "loop_state.pkl"), "wb"))

def _save_history_csv(epoch, logs, save_path, H):
    out = ""
    keys_to_save = []
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            out += "{key}={value}\t".format(key=key, value=value)
            keys_to_save.append(key)
    logger.info(out)
    logger.info("Saving history to " + os.path.join(save_path, "history.csv"))
    try:
        pd.DataFrame(H)[keys_to_save].to_csv(os.path.join(save_path, "history.csv"), index=False)
    except ValueError:
        logger.warning("Couldnt' save history.csv. Please read in history.pkl")

def _append_to_history_csv(epoch, logs, H):
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            if key not in H:
                H[key] = [value]
            else:
                H[key].append(value)

            assert len(H[key]) == epoch + 1, "Len H[{}] is {}, expected {} ".format(key, len(H[key]), epoch + 1)
        else:
            pass

def _reload(model, optimizer, save_path, callbacks):
    model_last_epoch_path = os.path.join(save_path, "model_last_epoch")
    loop_state_path = os.path.join(save_path, "loop_state.pkl")
    history_csv_path = os.path.join(save_path, "history.csv")

    if not os.path.exists(model_last_epoch_path) or not os.path.exists(loop_state_path):
        logger.warning("Failed to find last epoch model or loop state")
        return {}, 0

    # Reload everything (model, optimizer, loop state)
    logger.warning("Reloading weights!")
    model, optimizer = restore_model_and_optimizer(model, optimizer, model_last_epoch_path)
    logger.info("Reloading loop state!")
    loop_state = pickle.load(open(loop_state_path, 'rb'))
    logger.info("Reloading history!")
    H = pd.read_csv(history_csv_path)
    H = {col: list(H[col].values) for col in H.columns}
    print(H)
    logger.info("Done reloading!")

    # Small back-up
    os.system("cp " + os.path.join(save_path, "history.pkl") + " " + os.path.join(save_path, "history.pkl.bckp"))

    # Setup the rest
    epoch_start = loop_state['epochs_done'] + 1
    if not len(H[next(iter(H))]) == loop_state['epochs_done'] + 1:
        raise IOError("Mismatch between saved history and epochs recorded. "
                      "Found len(H)={0} and epoch_start={1} "
                      "Run was likely interrupted incorrectly and cannot be rerun.".format(len(H[next(iter(H))]),
                                                                                epoch_start))

    # Load all callbacks from the loop_state
    for e, e_loaded in zip(callbacks, loop_state['callbacks']):
        assert type(e) == type(e_loaded)
        if hasattr(e, "__setstate__"):
            e.__setstate__(e_loaded.__dict__)
        else:
            e.__dict__.update(e_loaded.__dict__)

    # Some diagnostics
    logger.info(loop_state)
    for k in H:
        logger.info((k, len(H)))
        break

    logger.info("epoch_start={}".format(epoch_start))

    return model, optimizer, H, epoch_start

def _to_infinite_iterator(it):
    while True:
        for bundle in it:
            yield bundle

def evaluate(model, data_generators, loss_function, metrics):
    results_all = defaultdict(float)
    for dataset_id in range(len(data_generators)):
        results = defaultdict(float)
        seen = 0

        for m in metrics:
            if isinstance(m, keras_metric):
                m.reset_states()

        for x, y in data_generators[dataset_id]:
            outputs = model.predict(x)

            if isinstance(x, dict):
                for k in x:
                    seen += len(x[k])
                    assert len(outputs) == len(x[k])
                    break
            else:
                seen += len(x)
                assert len(outputs) == len(x)

            results[f'loss:{dataset_id}'] += float(np.sum(loss_function(y, outputs)))

            for m in metrics:
                results[m.__name__ + ":" + str(dataset_id)] += float(np.sum(m(y, outputs)))

        for m in metrics:
            results[m.__name__ + ":" + str(dataset_id)] /= seen
        results_all.update(results)

    # Just to make sure
    for m in metrics:
        if isinstance(m, keras_metric):
            m.reset_states()

    return results_all

def _train_on_batch(model, optimizer, x_trains, y_trains, metrics, loss_function,
                    weight_decay=0):
    batch_logs = {}

    with tf.GradientTape(persistent=True) as tape:  # TODO: Fix persistency
        loss = 0
        all_outputs = []
        all_inputs = []
        for dataset_id, (x_train, y_train) in enumerate(zip(x_trains, y_trains)):
            outputs = model(x_train, training=True)

            all_outputs.append(outputs)
            all_inputs.append(x_train)

            loss += tf.math.reduce_mean(loss_function(y_train, outputs))

            # Update logs
            for m in metrics:
                if isinstance(m, str):
                    continue
                batch_logs[m.__name__ + ":" + str(dataset_id)] = tf.math.reduce_mean(
                    m(outputs, y_train))  # If not scalar, reduce it

            batch_logs[f'loss:{dataset_id}'] = loss

    grads = tape.gradient(loss, model.trainable_weights)

    if weight_decay > 0:
        grads = [g + weight_decay * p for (g, p) in zip(grads, model.trainable_weights)]

    optimizer.apply_gradients(zip(grads, model.trainable_variables))

    return batch_logs

@tf.function
def _train_on_batch_optimized(model, optimizer, x_trains, y_trains, metrics,
                              loss_function, weight_decay=0):
    return _train_on_batch(model=model,
                           optimizer=optimizer,
                           x_trains=x_trains, y_trains=y_trains,
                           metrics=metrics,
                           loss_function=loss_function, weight_decay=weight_decay)

def _training_loop(model, datasets, optimizer, loss_function, initial_epoch, epochs, callbacks,
                   steps_per_epoch, train_on_batch, evaluate_model, metrics=[], weight_decay=0,
                   evaluation_freq=1):
    """
    Internal implementation of the training loop.
    """
    tf.keras.backend.set_learning_phase(1)

    train_generators = [_to_infinite_iterator(d[0]) for d in datasets]
    valid_generators = [d[1] for d in datasets]

    for c in callbacks:
        c.on_train_begin(model)

    cumulative_batch_id = 0

    for epoch in range(initial_epoch, epochs):
        logger.info(f"Start epoch {epoch}")

        epoch_logs = {}

        for c in callbacks:
            c.on_epoch_begin(epoch, epoch_logs)
        for batch_id in range(steps_per_epoch):
            cumulative_batch_id += 1
            batch_logs = {}

            x_trains, y_trains, y_trains_a, y_trains_b, lams = [], [], [], [], []

            for dataset_id in range(len(datasets)):
                x_train, y_train = next(train_generators[dataset_id])

                x_trains.append(x_train)
                y_trains.append(y_train)

                if isinstance(x_train, dict):
                    batch_logs.update({"size:" + str(dataset_id): len(list(x_train.values())[0])})
                else:
                    batch_logs.update({"size:" + str(dataset_id): len(x_train)})

            for c in callbacks:
                c.on_batch_begin(batch=batch_id, logs=batch_logs)

            batch_logs_step = train_on_batch(model, optimizer, x_trains, y_trains,
                                             metrics,
                                             loss_function,
                                             weight_decay=weight_decay)

            batch_logs.update(batch_logs_step)
            for k in batch_logs:
                if hasattr(batch_logs[k], "numpy"):
                    batch_logs[k] = batch_logs[k].numpy()

                if (hasattr(batch_logs[k], 'ndim') and batch_logs[k].ndim > 0) or isinstance(batch_logs[k], list):
                    batch_logs[k] = batch_logs[k]  # .numpy()
                    if isinstance(batch_logs[k], list):
                        batch_logs[k] = np.array(batch_logs[k])
                else:
                    batch_logs[k] = float(batch_logs[k])

            for c in callbacks:
                c.on_batch_end(batch=batch_id, logs=batch_logs)

        if evaluation_freq > 0 and (epoch % evaluation_freq == 0 or epoch == epochs - 1):
            tf.keras.backend.set_learning_phase(0)
            val_results = evaluate_model(model, valid_generators, loss_function, metrics)
            tf.keras.backend.set_learning_phase(1)
            for k, v in val_results.items():
                epoch_logs[f'val_{k}'] = v
        else:
            if evaluation_freq > 0:
                for k in previous_epoch_logs:
                    if k not in epoch_logs:
                        epoch_logs[k] = np.nan

        for c in callbacks:
            c.on_epoch_end(epoch, epoch_logs)

        logger.info('End of epoch {}, loss={}'.format(epoch, epoch_logs['loss:0']))

        previous_epoch_logs = dict(epoch_logs)

    for c in callbacks:
        c.on_train_end(model)

@gin.configurable
def training_loop(model, loss_function, metrics, optimizer,
                  config, save_path, datasets, steps_per_epoch, seed,
                  custom_callbacks=[], checkpoint_monitor="val_categorical_accuracy:0",
                  use_tb=False, reload=False, evaluation_freq=1,
                  n_epochs=100, save_freq=1, save_history_every_k_examples=1,
                  load_weights_from="", load_weights_and_optimizer_from="",
                  weight_decay=0, load_classifier=True):
    if load_weights_and_optimizer_from != "":
        assert load_weights_from == ""
        assert load_weights_and_optimizer_from.endswith("h5")
        logger.info(f"load_weights_and_optimizer_from={load_weights_and_optimizer_from}")
        _, optimizer = restore_model_and_optimizer(model, optimizer, load_weights_and_optimizer_from)
        logger.info("Loaded optimizer")
        model.load_weights(load_weights_and_optimizer_from, by_name=True)
        model.optimizer = optimizer
    elif load_weights_from != "":
        assert load_weights_and_optimizer_from == ""
        model.load_weights(load_weights_from)

    if reload:
        assert load_classifier is True
        logger.warning("Only custom_callbacks can be reloaded for now")
        previous_model = model
        model, optimizer, H, epoch_start = _reload(model, optimizer, save_path, custom_callbacks)
        del previous_model
        logger.warning("Changed model reference internally in the training loop!")
    else:
        if hasattr(model, "compile"):
            model.compile(optimizer=optimizer,
                          loss=loss_function,
                          metrics=[m for m in metrics if not isinstance(m, str)])  # FIXME

        save_model(model, optimizer, os.path.join(save_path, "init_weights"))

        history_csv_path, history_pkl_path = os.path.join(save_path, "history.csv"), os.path.join(save_path,
                                                                                "history.pkl")
        logger.info("Removing {} and {}".format(history_pkl_path, history_csv_path))
        os.system("rm " + history_pkl_path)
        os.system("rm " + history_csv_path)
        H, epoch_start = {}, 0

    callbacks = list(custom_callbacks)
    callbacks += _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor,
                                              save_freq, custom_callbacks, use_tb,
                                              save_history_every_k_examples)

    # Configure callbacks
    for clbk in callbacks:
        clbk.set_save_path(save_path)
        clbk.set_optimizer(optimizer)
        clbk.set_model(model)
        clbk.set_seed(seed)
        clbk.set_datasets(datasets)
        clbk.set_config(config)
        clbk.set_callbacks(callbacks)

    _training_loop(model, datasets, optimizer, loss_function, epoch_start, n_epochs, callbacks,
                   steps_per_epoch, train_on_batch=_train_on_batch_optimized,
                   evaluate_model=evaluate,
                   metrics=metrics, evaluation_freq=evaluation_freq,
                   weight_decay=weight_decay)

    if save_freq != -1:
        save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))


for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            if key not in H:
                H[key] = [value]
            else:
                H[key].append(value)

            assert len(H[key]) == epoch + 1, "Len H[{}] is {}, expected {} ".format(key, len(H[key]), epoch + 1)
        else:
            pass

"""
Self-contained training loop for the project.
"""

import logging
import os
import pickle
from collections import defaultdict
from functools import partial
import gin
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow.keras.metrics import Metric as keras_metric

from src.callbacks import ModelCheckpoint, LambdaCallback, History, DumpTensorboardSummaries, BaseLogger, Heartbeat, \
    MetaSaver, MeasureTime
from src.utils import save_model, restore_model_and_optimizer, restore_model, configure_neptune_exp

logger = logging.getLogger(__name__)

def _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor, save_freq, custom_callbacks,
                                 use_tb, save_history_every_k_examples):
    callbacks = []
    callbacks.append(MetaSaver())
    callbacks.append(BaseLogger())
    callbacks.append(MeasureTime())
    callbacks.append(Heartbeat())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_append_to_history_csv, H=H)))
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_history_csv, save_path=save_path, H=H)))
    callbacks.append(History(save_every_k_examples=save_history_every_k_examples))

    if len(checkpoint_monitor):
        callbacks.append(ModelCheckpoint(monitor=checkpoint_monitor,
                                         save_best_only=True,
                                         mode='max',
                                         filepath=os.path.join(save_path, "model_best_val")))
    if save_freq > 0:
        def save_weights_fnc(epoch, logs):
            if epoch % save_freq == 0:
                logger.info("Saving model from epoch " + str(epoch))
                save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))

        callbacks.append(LambdaCallback(on_epoch_end=save_weights_fnc))

    if use_tb:
        callbacks.append(DumpTensorboardSummaries())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_loop_state, save_callbacks=custom_callbacks,
                                                         save_path=save_path)))
    return callbacks

def _save_loop_state(epoch, logs, save_path, save_callbacks):
    loop_state = {"epochs_done": epoch, "callbacks": save_callbacks}
    pickle.dump(loop_state, open(os.path.join(save_path, "loop_state.pkl"), "wb"))

def _save_history_csv(epoch, logs, save_path, H):
    out = ""
    keys_to_save = []
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            out += "{key}={value}\t".format(key=key, value=value)
            keys_to_save.append(key)
    logger.info(out)
    logger.info("Saving history to " + os.path.join(save_path, "history.csv"))
    try:
        pd.DataFrame(H)[keys_to_save].to_csv(os.path.join(save_path, "history.csv"), index=False)
    except ValueError:
        logger.warning("Couldnt' save history.csv. Please read in history.pkl")

def _append_to_history_csv(epoch, logs, H):

def _reload(model, optimizer, save_path, callbacks):
    model_last_epoch_path = os.path.join(save_path, "model_last_epoch")
    loop_state_path = os.path.join(save_path, "loop_state.pkl")
    history_csv_path = os.path.join(save_path, "history.csv")

    if not os.path.exists(model_last_epoch_path) or not os.path.exists(loop_state_path):
        logger.warning("Failed to find last epoch model or loop state")
        return {}, 0

    # Reload everything (model, optimizer, loop state)
    logger.warning("Reloading weights!")
    model, optimizer = restore_model_and_optimizer(model, optimizer, model_last_epoch_path)
    logger.info("Reloading loop state!")
    loop_state = pickle.load(open(loop_state_path, 'rb'))
    logger.info("Reloading history!")
    H = pd.read_csv(history_csv_path)
    H = {col: list(H[col].values) for col in H.columns}
    print(H)
    logger.info("Done reloading!")

    # Small back-up
    os.system("cp " + os.path.join(save_path, "history.pkl") + " " + os.path.join(save_path, "history.pkl.bckp"))

    # Setup the rest
    epoch_start = loop_state['epochs_done'] + 1
    if not len(H[next(iter(H))]) == loop_state['epochs_done'] + 1:
        raise IOError("Mismatch between saved history and epochs recorded. "
                      "Found len(H)={0} and epoch_start={1} "
                      "Run was likely interrupted incorrectly and cannot be rerun.".format(len(H[next(iter(H))]),
                                                                                epoch_start))

    # Load all callbacks from the loop_state
    for e, e_loaded in zip(callbacks, loop_state['callbacks']):
        assert type(e) == type(e_loaded)
        if hasattr(e, "__setstate__"):
            e.__setstate__(e_loaded.__dict__)
        else:
            e.__dict__.update(e_loaded.__dict__)

    # Some diagnostics
    logger.info(loop_state)
    for k in H:
        logger.info((k, len(H)))
        break

    logger.info("epoch_start={}".format(epoch_start))

    return model, optimizer, H, epoch_start

def _to_infinite_iterator(it):
    while True:
        for bundle in it:
            yield bundle

def evaluate(model, data_generators, loss_function, metrics):
    results_all = defaultdict(float)
    for dataset_id in range(len(data_generators)):
        results = defaultdict(float)
        seen = 0

        for m in metrics:
            if isinstance(m, keras_metric):
                m.reset_states()

        for x, y in data_generators[dataset_id]:
            outputs = model.predict(x)

            if isinstance(x, dict):
                for k in x:
                    seen += len(x[k])
                    assert len(outputs) == len(x[k])
                    break
            else:
                seen += len(x)
                assert len(outputs) == len(x)

            results[f'loss:{dataset_id}'] += float(np.sum(loss_function(y, outputs)))

            for m in metrics:
                results[m.__name__ + ":" + str(dataset_id)] += float(np.sum(m(y, outputs)))

        for m in metrics:
            results[m.__name__ + ":" + str(dataset_id)] /= seen
        results_all.update(results)

    # Just to make sure
    for m in metrics:
        if isinstance(m, keras_metric):
            m.reset_states()

    return results_all

def _train_on_batch(model, optimizer, x_trains, y_trains, metrics, loss_function,
                    weight_decay=0):
    batch_logs = {}

    with tf.GradientTape(persistent=True) as tape:  # TODO: Fix persistency
        loss = 0
        all_outputs = []
        all_inputs = []
        for dataset_id, (x_train, y_train) in enumerate(zip(x_trains, y_trains)):
            outputs = model(x_train, training=True)

            all_outputs.append(outputs)
            all_inputs.append(x_train)

            loss += tf.math.reduce_mean(loss_function(y_train, outputs))

            # Update logs
            for m in metrics:
                if isinstance(m, str):
                    continue
                batch_logs[m.__name__ + ":" + str(dataset_id)] = tf.math.reduce_mean(
                    m(outputs, y_train))  # If not scalar, reduce it

            batch_logs[f'loss:{dataset_id}'] = loss

    grads = tape.gradient(loss, model.trainable_weights)

    if weight_decay > 0:
        grads = [g + weight_decay * p for (g, p) in zip(grads, model.trainable_weights)]

    optimizer.apply_gradients(zip(grads, model.trainable_variables))

    return batch_logs

@tf.function
def _train_on_batch_optimized(model, optimizer, x_trains, y_trains, metrics,
                              loss_function, weight_decay=0):
    return _train_on_batch(model=model,
                           optimizer=optimizer,
                           x_trains=x_trains, y_trains=y_trains,
                           metrics=metrics,
                           loss_function=loss_function, weight_decay=weight_decay)

def _training_loop(model, datasets, optimizer, loss_function, initial_epoch, epochs, callbacks,
                   steps_per_epoch, train_on_batch, evaluate_model, metrics=[], weight_decay=0,
                   evaluation_freq=1):
    """
    Internal implementation of the training loop.
    """
    tf.keras.backend.set_learning_phase(1)

    train_generators = [_to_infinite_iterator(d[0]) for d in datasets]
    valid_generators = [d[1] for d in datasets]

    for c in callbacks:
        c.on_train_begin(model)

    cumulative_batch_id = 0

    for epoch in range(initial_epoch, epochs):
        logger.info(f"Start epoch {epoch}")

        epoch_logs = {}

        for c in callbacks:
            c.on_epoch_begin(epoch, epoch_logs)
        for batch_id in range(steps_per_epoch):
            cumulative_batch_id += 1
            batch_logs = {}

            x_trains, y_trains, y_trains_a, y_trains_b, lams = [], [], [], [], []

            for dataset_id in range(len(datasets)):
                x_train, y_train = next(train_generators[dataset_id])

                x_trains.append(x_train)
                y_trains.append(y_train)

                if isinstance(x_train, dict):
                    batch_logs.update({"size:" + str(dataset_id): len(list(x_train.values())[0])})
                else:
                    batch_logs.update({"size:" + str(dataset_id): len(x_train)})

            for c in callbacks:
                c.on_batch_begin(batch=batch_id, logs=batch_logs)

            batch_logs_step = train_on_batch(model, optimizer, x_trains, y_trains,
                                             metrics,
                                             loss_function,
                                             weight_decay=weight_decay)

            batch_logs.update(batch_logs_step)
            for k in batch_logs:
                if hasattr(batch_logs[k], "numpy"):
                    batch_logs[k] = batch_logs[k].numpy()

                if (hasattr(batch_logs[k], 'ndim') and batch_logs[k].ndim > 0) or isinstance(batch_logs[k], list):
                    batch_logs[k] = batch_logs[k]  # .numpy()
                    if isinstance(batch_logs[k], list):
                        batch_logs[k] = np.array(batch_logs[k])
                else:
                    batch_logs[k] = float(batch_logs[k])

            for c in callbacks:
                c.on_batch_end(batch=batch_id, logs=batch_logs)

        if evaluation_freq > 0 and (epoch % evaluation_freq == 0 or epoch == epochs - 1):
            tf.keras.backend.set_learning_phase(0)
            val_results = evaluate_model(model, valid_generators, loss_function, metrics)
            tf.keras.backend.set_learning_phase(1)
            for k, v in val_results.items():
                epoch_logs[f'val_{k}'] = v
        else:
            if evaluation_freq > 0:
                for k in previous_epoch_logs:
                    if k not in epoch_logs:
                        epoch_logs[k] = np.nan

        for c in callbacks:
            c.on_epoch_end(epoch, epoch_logs)

        logger.info('End of epoch {}, loss={}'.format(epoch, epoch_logs['loss:0']))

        previous_epoch_logs = dict(epoch_logs)

    for c in callbacks:
        c.on_train_end(model)

@gin.configurable
def training_loop(model, loss_function, metrics, optimizer,
                  config, save_path, datasets, steps_per_epoch, seed,
                  custom_callbacks=[], checkpoint_monitor="val_categorical_accuracy:0",
                  use_tb=False, reload=False, evaluation_freq=1,
                  n_epochs=100, save_freq=1, save_history_every_k_examples=1,
                  load_weights_from="", load_weights_and_optimizer_from="",
                  weight_decay=0, load_classifier=True):
    if load_weights_and_optimizer_from != "":
        assert load_weights_from == ""
        assert load_weights_and_optimizer_from.endswith("h5")
        logger.info(f"load_weights_and_optimizer_from={load_weights_and_optimizer_from}")
        _, optimizer = restore_model_and_optimizer(model, optimizer, load_weights_and_optimizer_from)
        logger.info("Loaded optimizer")
        model.load_weights(load_weights_and_optimizer_from, by_name=True)
        model.optimizer = optimizer
    elif load_weights_from != "":
        assert load_weights_and_optimizer_from == ""
        model.load_weights(load_weights_from)

    if reload:
        assert load_classifier is True
        logger.warning("Only custom_callbacks can be reloaded for now")
        previous_model = model
        model, optimizer, H, epoch_start = _reload(model, optimizer, save_path, custom_callbacks)
        del previous_model
        logger.warning("Changed model reference internally in the training loop!")
    else:
        if hasattr(model, "compile"):
            model.compile(optimizer=optimizer,
                          loss=loss_function,
                          metrics=[m for m in metrics if not isinstance(m, str)])  # FIXME

        save_model(model, optimizer, os.path.join(save_path, "init_weights"))

        history_csv_path, history_pkl_path = os.path.join(save_path, "history.csv"), os.path.join(save_path,
                                                                                "history.pkl")
        logger.info("Removing {} and {}".format(history_pkl_path, history_csv_path))
        os.system("rm " + history_pkl_path)
        os.system("rm " + history_csv_path)
        H, epoch_start = {}, 0

    callbacks = list(custom_callbacks)
    callbacks += _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor,
                                              save_freq, custom_callbacks, use_tb,
                                              save_history_every_k_examples)

    # Configure callbacks
    for clbk in callbacks:
        clbk.set_save_path(save_path)
        clbk.set_optimizer(optimizer)
        clbk.set_model(model)
        clbk.set_seed(seed)
        clbk.set_datasets(datasets)
        clbk.set_config(config)
        clbk.set_callbacks(callbacks)

    _training_loop(model, datasets, optimizer, loss_function, epoch_start, n_epochs, callbacks,
                   steps_per_epoch, train_on_batch=_train_on_batch_optimized,
                   evaluate_model=evaluate,
                   metrics=metrics, evaluation_freq=evaluation_freq,
                   weight_decay=weight_decay)

    if save_freq != -1:
        save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))


"""
Self-contained training loop for the project.
"""

import logging
import os
import pickle
from collections import defaultdict
from functools import partial
import gin
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow.keras.metrics import Metric as keras_metric

from src.callbacks import ModelCheckpoint, LambdaCallback, History, DumpTensorboardSummaries, BaseLogger, Heartbeat, \
    MetaSaver, MeasureTime
from src.utils import save_model, restore_model_and_optimizer, restore_model, configure_neptune_exp

logger = logging.getLogger(__name__)

def _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor, save_freq, custom_callbacks,
                                 use_tb, save_history_every_k_examples):
    callbacks = []
    callbacks.append(MetaSaver())
    callbacks.append(BaseLogger())
    callbacks.append(MeasureTime())
    callbacks.append(Heartbeat())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_append_to_history_csv, H=H)))
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_history_csv, save_path=save_path, H=H)))
    callbacks.append(History(save_every_k_examples=save_history_every_k_examples))

    if len(checkpoint_monitor):
        callbacks.append(ModelCheckpoint(monitor=checkpoint_monitor,
                                         save_best_only=True,
                                         mode='max',
                                         filepath=os.path.join(save_path, "model_best_val")))
    if save_freq > 0:
        def save_weights_fnc(epoch, logs):
            if epoch % save_freq == 0:
                logger.info("Saving model from epoch " + str(epoch))
                save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))

        callbacks.append(LambdaCallback(on_epoch_end=save_weights_fnc))

    if use_tb:
        callbacks.append(DumpTensorboardSummaries())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_loop_state, save_callbacks=custom_callbacks,
                                                         save_path=save_path)))
    return callbacks

def _save_loop_state(epoch, logs, save_path, save_callbacks):
    loop_state = {"epochs_done": epoch, "callbacks": save_callbacks}
    pickle.dump(loop_state, open(os.path.join(save_path, "loop_state.pkl"), "wb"))

def _save_history_csv(epoch, logs, save_path, H):
    out = ""
    keys_to_save = []
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            out += "{key}={value}\t".format(key=key, value=value)
            keys_to_save.append(key)
    logger.info(out)
    logger.info("Saving history to " + os.path.join(save_path, "history.csv"))
    try:
        pd.DataFrame(H)[keys_to_save].to_csv(os.path.join(save_path, "history.csv"), index=False)
    except ValueError:
        logger.warning("Couldnt' save history.csv. Please read in history.pkl")

def _append_to_history_csv(epoch, logs, H):
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            if key not in H:
                H[key] = [value]
            else:
                H[key].append(value)

            assert len(H[key]) == epoch + 1, "Len H[{}] is {}, expected {} ".format(key, len(H[key]), epoch + 1)
        else:
            pass

def _reload(model, optimizer, save_path, callbacks):
    model_last_epoch_path = os.path.join(save_path, "model_last_epoch")
    loop_state_path = os.path.join(save_path, "loop_state.pkl")
    history_csv

    if not os.path.exists(model_last_epoch_path) or not os.path.exists(loop_state_path):
        logger.warning("Failed to find last epoch model or loop state")
        return {}, 0

    # Reload everything (model, optimizer, loop state)
    logger.warning("Reloading weights!")
    model, optimizer = restore_model_and_optimizer(model, optimizer, model_last_epoch_path)
    logger.info("Reloading loop state!")
    loop_state = pickle.load(open(loop_state_path, 'rb'))
    logger.info("Reloading history!")
    H = pd.read_csv(history_csv_path)
    H = {col: list(H[col].values) for col in H.columns}
    print(H)
    logger.info("Done reloading!")

    # Small back-up
    os.system("cp " + os.path.join(save_path, "history.pkl") + " " + os.path.join(save_path, "history.pkl.bckp"))

    # Setup the rest
    epoch_start = loop_state['epochs_done'] + 1
    if not len(H[next(iter(H))]) == loop_state['epochs_done'] + 1:
        raise IOError("Mismatch between saved history and epochs recorded. "
                      "Found len(H)={0} and epoch_start={1} "
                      "Run was likely interrupted incorrectly and cannot be rerun.".format(len(H[next(iter(H))]),
                                                                                epoch_start))

    # Load all callbacks from the loop_state
    for e, e_loaded in zip(callbacks, loop_state['callbacks']):
        assert type(e) == type(e_loaded)
        if hasattr(e, "__setstate__"):
            e.__setstate__(e_loaded.__dict__)
        else:
            e.__dict__.update(e_loaded.__dict__)

    # Some diagnostics
    logger.info(loop_state)
    for k in H:
        logger.info((k, len(H)))
        break

    logger.info("epoch_start={}".format(epoch_start))

    return model, optimizer, H, epoch_start

def _to_infinite_iterator(it):
    while True:
        for bundle in it:
            yield bundle

def evaluate(model, data_generators, loss_function, metrics):
    results_all = defaultdict(float)
    for dataset_id in range(len(data_generators)):
        results = defaultdict(float)
        seen = 0

        for m in metrics:
            if isinstance(m, keras_metric):
                m.reset_states()

        for x, y in data_generators[dataset_id]:
            outputs = model.predict(x)

            if isinstance(x, dict):
                for k in x:
                    seen += len(x[k])
                    assert len(outputs) == len(x[k])
                    break
            else:
                seen += len(x)
                assert len(outputs) == len(x)

            results[f'loss:{dataset_id}'] += float(np.sum(loss_function(y, outputs)))

            for m in metrics:
                results[m.__name__ + ":" + str(dataset_id)] += float(np.sum(m(y, outputs)))

        for m in metrics:
            results[m.__name__ + ":" + str(dataset_id)] /= seen
        results_all.update(results)

    # Just to make sure
    for m in metrics:
        if isinstance(m, keras_metric):
            m.reset_states()

    return results_all

def _train_on_batch(model, optimizer, x_trains, y_trains, metrics, loss_function,
                    weight_decay=0):
    batch_logs = {}

    with tf.GradientTape(persistent=True) as tape:  # TODO: Fix persistency
        loss = 0
        all_outputs = []
        all_inputs = []
        for dataset_id, (x_train, y_train) in enumerate(zip(x_trains, y_trains)):
            outputs = model(x_train, training=True)

            all_outputs.append(outputs)
            all_inputs.append(x_train)

            loss += tf.math.reduce_mean(loss_function(y_train, outputs))

            # Update logs
            for m in metrics:
                if isinstance(m, str):
                    continue
                batch_logs[m.__name__ + ":" + str(dataset_id)] = tf.math.reduce_mean(
                    m(outputs, y_train))  # If not scalar, reduce it

            batch_logs[f'loss:{dataset_id}'] = loss

    grads = tape.gradient(loss, model.trainable_weights)

    if weight_decay > 0:
        grads = [g + weight_decay * p for (g, p) in zip(grads, model.trainable_weights)]

    optimizer.apply_gradients(zip(grads, model.trainable_variables))

    return batch_logs

@tf.function
def _train_on_batch_optimized(model, optimizer, x_trains, y_trains, metrics,
                              loss_function, weight_decay=0):
    return _train_on_batch(model=model,
                           optimizer=optimizer,
                           x_trains=x_trains, y_trains=y_trains,
                           metrics=metrics,
                           loss_function=loss_function, weight_decay=weight_decay)

def _training_loop(model, datasets, optimizer, loss_function, initial_epoch, epochs, callbacks,
                   steps_per_epoch, train_on_batch, evaluate_model, metrics=[], weight_decay=0,
                   evaluation_freq=1):
    """
    Internal implementation of the training loop.
    """
    tf.keras.backend.set_learning_phase(1)

    train_generators = [_to_infinite_iterator(d[0]) for d in datasets]
    valid_generators = [d[1] for d in datasets]

    for c in callbacks:
        c.on_train_begin(model)

    cumulative_batch_id = 0

    for epoch in range(initial_epoch, epochs):
        logger.info(f"Start epoch {epoch}")

        epoch_logs = {}

        for c in callbacks:
            c.on_epoch_begin(epoch, epoch_logs)
        for batch_id in range(steps_per_epoch):
            cumulative_batch_id += 1
            batch_logs = {}

            x_trains, y_trains, y_trains_a, y_trains_b, lams = [], [], [], [], []

            for dataset_id in range(len(datasets)):
                x_train, y_train = next(train_generators[dataset_id])

                x_trains.append(x_train)
                y_trains.append(y_train)

                if isinstance(x_train, dict):
                    batch_logs.update({"size:" + str(dataset_id): len(list(x_train.values())[0])})
                else:
                    batch_logs.update({"size:" + str(dataset_id): len(x_train)})

            for c in callbacks:
                c.on_batch_begin(batch=batch_id, logs=batch_logs)

            batch_logs_step = train_on_batch(model, optimizer, x_trains, y_trains,
                                             metrics,
                                             loss_function,
                                             weight_decay=weight_decay)

            batch_logs.update(batch_logs_step)
            for k in batch_logs:
                if hasattr(batch_logs[k], "numpy"):
                    batch_logs[k] = batch_logs[k].numpy()

                if (hasattr(batch_logs[k], 'ndim') and batch_logs[k].ndim > 0) or isinstance(batch_logs[k], list):
                    batch_logs[k] = batch_logs[k]  # .numpy()
                    if isinstance(batch_logs[k], list):
                        batch_logs[k] = np.array(batch_logs[k])
                else:
                    batch_logs[k] = float(batch_logs[k])

            for c in callbacks:
                c.on_batch_end(batch=batch_id, logs=batch_logs)

        if evaluation_freq > 0 and (epoch % evaluation_freq == 0 or epoch == epochs - 1):
            tf.keras.backend.set_learning_phase(0)
            val_results = evaluate_model(model, valid_generators, loss_function, metrics)
            tf.keras.backend.set_learning_phase(1)
            for k, v in val_results.items():
                epoch_logs[f'val_{k}'] = v
        else:
            if evaluation_freq > 0:
                for k in previous_epoch_logs:
                    if k not in epoch_logs:
                        epoch_logs[k] = np.nan

        for c in callbacks:
            c.on_epoch_end(epoch, epoch_logs)

        logger.info('End of epoch {}, loss={}'.format(epoch, epoch_logs['loss:0']))

        previous_epoch_logs = dict(epoch_logs)

    for c in callbacks:
        c.on_train_end(model)

@gin.configurable
def training_loop(model, loss_function, metrics, optimizer,
                  config, save_path, datasets, steps_per_epoch, seed,
                  custom_callbacks=[], checkpoint_monitor="val_categorical_accuracy:0",
                  use_tb=False, reload=False, evaluation_freq=1,
                  n_epochs=100, save_freq=1, save_history_every_k_examples=1,
                  load_weights_from="", load_weights_and_optimizer_from="",
                  weight_decay=0, load_classifier=True):
    if load_weights_and_optimizer_from != "":
        assert load_weights_from == ""
        assert load_weights_and_optimizer_from.endswith("h5")
        logger.info(f"load_weights_and_optimizer_from={load_weights_and_optimizer_from}")
        _, optimizer = restore_model_and_optimizer(model, optimizer, load_weights_and_optimizer_from)
        logger.info("Loaded optimizer")
        model.load_weights(load_weights_and_optimizer_from, by_name=True)
        model.optimizer = optimizer
    elif load_weights_from != "":
        assert load_weights_and_optimizer_from == ""
        model.load_weights(load_weights_from)

    if reload:
        assert load_classifier is True
        logger.warning("Only custom_callbacks can be reloaded for now")
        previous_model = model
        model, optimizer, H, epoch_start = _reload(model, optimizer, save_path, custom_callbacks)
        del previous_model
        logger.warning("Changed model reference internally in the training loop!")
    else:
        if hasattr(model, "compile"):
            model.compile(optimizer=optimizer,
                          loss=loss_function,
                          metrics=[m for m in metrics if not isinstance(m, str)])  # FIXME

        save_model(model, optimizer, os.path.join(save_path, "init_weights"))

        history_csv_path, history_pkl_path = os.path.join(save_path, "history.csv"), os.path.join(save_path,
                                                                                "history.pkl")
        logger.info("Removing {} and {}".format(history_pkl_path, history_csv_path))
        os.system("rm " + history_pkl_path)
        os.system("rm " + history_csv_path)
        H, epoch_start = {}, 0

    callbacks = list(custom_callbacks)
    callbacks += _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor,
                                              save_freq, custom_callbacks, use_tb,
                                              save_history_every_k_examples)

    # Configure callbacks
    for clbk in callbacks:
        clbk.set_save_path(save_path)
        clbk.set_optimizer(optimizer)
        clbk.set_model(model)
        clbk.set_seed(seed)
        clbk.set_datasets(datasets)
        clbk.set_config(config)
        clbk.set_callbacks(callbacks)

    _training_loop(model, datasets, optimizer, loss_function, epoch_start, n_epochs, callbacks,
                   steps_per_epoch, train_on_batch=_train_on_batch_optimized,
                   evaluate_model=evaluate,
                   metrics=metrics, evaluation_freq=evaluation_freq,
                   weight_decay=weight_decay)

    if save_freq != -1:
        save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))


for c in callbacks:
                c.on_batch_begin(batch=batch_id, logs=batch_logs)

"""
Self-contained training loop for the project.
"""

import logging
import os
import pickle
from collections import defaultdict
from functools import partial
import gin
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow.keras.metrics import Metric as keras_metric

from src.callbacks import ModelCheckpoint, LambdaCallback, History, DumpTensorboardSummaries, BaseLogger, Heartbeat, \
    MetaSaver, MeasureTime
from src.utils import save_model, restore_model_and_optimizer, restore_model, configure_neptune_exp

logger = logging.getLogger(__name__)

def _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor, save_freq, custom_callbacks,
                                 use_tb, save_history_every_k_examples):
    callbacks = []
    callbacks.append(MetaSaver())
    callbacks.append(BaseLogger())
    callbacks.append(MeasureTime())
    callbacks.append(Heartbeat())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_append_to_history_csv, H=H)))
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_history_csv, save_path=save_path, H=H)))
    callbacks.append(History(save_every_k_examples=save_history_every_k_examples))

    if len(checkpoint_monitor):
        callbacks.append(ModelCheckpoint(monitor=checkpoint_monitor,
                                         save_best_only=True,
                                         mode='max',
                                         filepath=os.path.join(save_path, "model_best_val")))
    if save_freq > 0:
        def save_weights_fnc(epoch, logs):
            if epoch % save_freq == 0:
                logger.info("Saving model from epoch " + str(epoch))
                save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))

        callbacks.append(LambdaCallback(on_epoch_end=save_weights_fnc))

    if use_tb:
        callbacks.append(DumpTensorboardSummaries())
    callbacks.append(LambdaCallback(on_epoch_end=partial(_save_loop_state, save_callbacks=custom_callbacks,
                                                         save_path=save_path)))
    return callbacks

def _save_loop_state(epoch, logs, save_path, save_callbacks):
    loop_state = {"epochs_done": epoch, "callbacks": save_callbacks}
    pickle.dump(loop_state, open(os.path.join(save_path, "loop_state.pkl"), "wb"))

def _save_history_csv(epoch, logs, save_path, H):
    out = ""
    keys_to_save = []
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            out += "{key}={value}\t".format(key=key, value=value)
            keys_to_save.append(key)
    logger.info(out)
    logger.info("Saving history to " + os.path.join(save_path, "history.csv"))
    try:
        pd.DataFrame(H)[keys_to_save].to_csv(os.path.join(save_path, "history.csv"), index=False)
    except ValueError:
        logger.warning("Couldnt' save history.csv. Please read in history.pkl")

def _append_to_history_csv(epoch, logs, H):
    for key, value in logs.items():
        if isinstance(value, (int, float, complex, np.float32, np.float64)):
            if key not in H:
                H[key] = [value]
            else:
                H[key].append(value)

            assert len(H[key]) == epoch + 1, "Len H[{}] is {}, expected {} ".format(key, len(H[key]), epoch + 1)
        else:
            pass

def _reload(model, optimizer, save_path, callbacks):
    model_last_epoch_path = os.path.join(save_path, "model_last_epoch")
    loop_state_path = os.path.join(save_path, "loop_state.pkl")
    history_csv_path = os.path.join(save_path, "history.csv")

    if not os.path.exists(model_last_epoch_path) or not os.path.exists(loop_state_path):
        logger.warning("Failed to find last epoch model or loop state")
        return {}, 0

    # Reload everything (model, optimizer, loop state)
    logger.warning("Reloading weights!")
    model, optimizer = restore_model_and_optimizer(model, optimizer, model_last_epoch_path)
    logger.info("Reloading loop state!")
    loop_state = pickle.load(open(loop_state_path, 'rb'))
    logger.info("Reloading history!")
    H = pd.read_csv(history_csv_path)
    H = {col: list(H[col].values) for col in H.columns}
    print(H)
    logger.info("Done reloading!")

    # Small back-up
    os.system("cp " + os.path.join(save_path, "history.pkl") + " " + os.path.join(save_path, "history.pkl.bckp"))

    # Setup the rest
    epoch_start = loop_state['epochs_done'] + 1
    if not len(H[next(iter(H))]) == loop_state['epochs_done'] + 1:
        raise IOError("Mismatch between saved history and epochs recorded. "
                      "Found len(H)={0} and epoch_start={1} "
                      "Run was likely interrupted incorrectly and cannot be rerun.".format(len(H[next(iter(H))]),
                                                                                epoch_start))

    # Load all callbacks from the loop_state
    for e, e_loaded in zip(callbacks, loop_state['callbacks']):
        assert type(e) == type(e_loaded)
        if hasattr(e, "__setstate__"):
            e.__setstate__(e_loaded.__dict__)
        else:
            e.__dict__.update(e_loaded.__dict__)

    # Some diagnostics
    logger.info(loop_state)
    for k in H:
        logger.info((k, len(H)))
        break

    logger.info("epoch_start={}".format(epoch_start))

    return model, optimizer, H, epoch_start

def _to_infinite_iterator(it):
    while True:
        for bundle in it:
            yield bundle

def evaluate(model, data_generators, loss_function, metrics):
    results_all = defaultdict(float)
    for dataset_id in range(len(data_generators)):
        results = defaultdict(float)
        seen = 0

        for m in metrics:
            if isinstance(m, keras_metric):
                m.reset_states()

        for x, y in data_generators[dataset_id]:
            outputs = model.predict(x)

            if isinstance(x, dict):
                for k in x:
                    seen += len(x[k])
                    assert len(outputs) == len(x[k])
                    break
            else:
                seen += len(x)
                assert len(outputs) == len(x)

            results[f'loss:{dataset_id}'] += float(np.sum(loss_function(y, outputs)))

            for m in metrics:
                results[m.__name__ + ":" + str(dataset_id)] += float(np.sum(m(y, outputs)))

        for m in metrics:
            results[m.__name__ + ":" + str(dataset_id)] /= seen
        results_all.update(results)

    # Just to make sure
    for m in metrics:
        if isinstance(m, keras_metric):
            m.reset_states()

    return results_all

def _train_on_batch(model, optimizer, x_trains, y_trains, metrics, loss_function,
                    weight_decay=0):
    batch_logs = {}

    with tf.GradientTape(persistent=True) as tape:  # TODO: Fix persistency
        loss = 0
        all_outputs = []
        all_inputs = []
        for dataset_id, (x_train, y_train) in enumerate(zip(x_trains, y_trains)):
            outputs = model(x_train, training=True)

            all_outputs.append(outputs)
            all_inputs.append(x_train)

            loss += tf.math.reduce_mean(loss_function(y_train, outputs))

            # Update logs
            for m in metrics:
                if isinstance(m, str):
                    continue
                batch_logs[m.__name__ + ":" + str(dataset_id)] = tf.math.reduce_mean(
                    m(outputs, y_train))  # If not scalar, reduce it

            batch_logs[f'loss:{dataset_id}'] = loss

    grads = tape.gradient(loss, model.trainable_weights)

    if weight_decay > 0:
        grads = [g + weight_decay * p for (g, p) in zip(grads, model.trainable_weights)]

    optimizer.apply_gradients(zip(grads, model.trainable_variables))

    return batch_logs

@tf.function
def _train_on_batch_optimized(model, optimizer, x_trains, y_trains, metrics,
                              loss_function, weight_decay=0):
    return _train_on_batch(model=model,
                           optimizer=optimizer,
                           x_trains=x_trains, y_trains=y_trains,
                           metrics=metrics,
                           loss_function=loss_function, weight_decay=weight_decay)

def _training_loop(model, datasets, optimizer, loss_function, initial_epoch, epochs, callbacks,
                   steps_per_epoch, train_on_batch, evaluate_model, metrics=[], weight_decay=0,
                   evaluation_freq=1):
    """
    Internal implementation of the training loop.
    """
    tf.keras.backend.set_learning_phase(1)

    train_generators = [_to_infinite_iterator(d[0]) for d in datasets]
    valid_generators = [d[1] for d in datasets]

    for c in callbacks:
        c.on_train_begin(model)

    cumulative_batch_id = 0

    for epoch in range(initial_epoch, epochs):
        logger.info(f"Start epoch {epoch}")

        epoch_logs = {}

        for c in callbacks:
            c.on_epoch_begin(epoch, epoch_logs)
        for batch_id in range(steps_per_epoch):
            cumulative_batch_id += 1
            batch_logs = {}

            x_trains, y_trains, y_trains_a, y_trains_b, lams = [], [], [], [], []

            for dataset_id in range(len(datasets)):
                x_train, y_train = next(train_generators[dataset_id])

                x_trains.append(x_train)
                y_trains.append(y_train)

                if isinstance(x_train, dict):
                    batch_logs.update({"size:" + str(dataset_id): len(list(x_train.values())[0])})
                else:
                    batch_logs.update({"size:" + str(dataset_id): len(x_train)})

            batch_logs_step = train_on_batch(model, optimizer, x_trains, y_trains,
                                             metrics,
                                             loss_function,
                                             weight_decay=weight_decay)

            batch_logs.update(batch_logs_step)
            for k in batch_logs:
                if hasattr(batch_logs[k], "numpy"):
                    batch_logs[k] = batch_logs[k].numpy()

                if (hasattr(batch_logs[k], 'ndim') and batch_logs[k].ndim > 0) or isinstance(batch_logs[k], list):
                    batch_logs[k] = batch_logs[k]  # .numpy()
                    if isinstance(batch_logs[k], list):
                        batch_logs[k] = np.array(batch_logs[k])
                else:
                    batch_logs[k] = float(batch_logs[k])

            for c in callbacks:
                c.on_batch_end(batch=batch_id, logs=batch_logs)

        if evaluation_freq > 0 and (epoch % evaluation_freq == 0 or epoch == epochs - 1):
            tf.keras.backend.set_learning_phase(0)
            val_results = evaluate_model(model, valid_generators, loss_function, metrics)
            tf.keras.backend.set_learning_phase(1)
            for k, v in val_results.items():
                epoch_logs[f'val_{k}'] = v
        else:
            if evaluation_freq > 0:
                for k in previous_epoch_logs:
                    if k not in epoch_logs:
                        epoch_logs[k] = np.nan

        for c in callbacks:
            c.on_epoch_end(epoch, epoch_logs)

        logger.info('End of epoch {}, loss={}'.format(epoch, epoch_logs['loss:0']))

        previous_epoch_logs = dict(epoch_logs)

    for c in callbacks:
        c.on_train_end(model)

@gin.configurable
def training_loop(model, loss_function, metrics, optimizer,
                  config, save_path, datasets, steps_per_epoch, seed,
                  custom_callbacks=[], checkpoint_monitor="val_categorical_accuracy:0",
                  use_tb=False, reload=False, evaluation_freq=1,
                  n_epochs=100, save_freq=1, save_history_every_k_examples=1,
                  load_weights_from="", load_weights_and_optimizer_from="",
                  weight_decay=0, load_classifier=True):
    if load_weights_and_optimizer_from != "":
        assert load_weights_from == ""
        assert load_weights_and_optimizer_from.endswith("h5")
        logger.info(f"load_weights_and_optimizer_from={load_weights_and_optimizer_from}")
        _, optimizer = restore_model_and_optimizer(model, optimizer, load_weights_and_optimizer_from)
        logger.info("Loaded optimizer")
        model.load_weights(load_weights_and_optimizer_from, by_name=True)
        model.optimizer = optimizer
    elif load_weights_from != "":
        assert load_weights_and_optimizer_from == ""
        model.load_weights(load_weights_from)

    if reload:
        assert load_classifier is True
        logger.warning("Only custom_callbacks can be reloaded for now")
        previous_model = model
        model, optimizer, H, epoch_start = _reload(model, optimizer, save_path, custom_callbacks)
        del previous_model
        logger.warning("Changed model reference internally in the training loop!")
    else:
        if hasattr(model, "compile"):
            model.compile(optimizer=optimizer,
                          loss=loss_function,
                          metrics=[m for m in metrics if not isinstance(m, str)])  # FIXME

        save_model(model, optimizer, os.path.join(save_path, "init_weights"))

        history_csv_path, history_pkl_path = os.path.join(save_path, "history.csv"), os.path.join(save_path,
                                                                                "history.pkl")
        logger.info("Removing {} and {}".format(history_pkl_path, history_csv_path))
        os.system("rm " + history_pkl_path)
        os.system("rm " + history_csv_path)
        H, epoch_start = {}, 0

    callbacks = list(custom_callbacks)
    callbacks += _construct_default_callbacks(model, optimizer, H, save_path, checkpoint_monitor,
                                              save_freq, custom_callbacks, use_tb,
                                              save_history_every_k_examples)

    # Configure callbacks
    for clbk in callbacks:
        clbk.set_save_path(save_path)
        clbk.set_optimizer(optimizer)
        clbk.set_model(model)
        clbk.set_seed(seed)
        clbk.set_datasets(datasets)
        clbk.set_config(config)
        clbk.set_callbacks(callbacks)

    _training_loop(model, datasets, optimizer, loss_function, epoch_start, n_epochs, callbacks,
                   steps_per_epoch, train_on_batch=_train_on_batch_optimized,
                   evaluate_model=evaluate,
                   metrics=metrics, evaluation_freq=evaluation_freq,
                   weight_decay=weight_decay)

    if save_freq != -1:
        save_model(model, optimizer, os.path.join(save_path, "model_last_epoch"))