try:
                msg = msg % args
            except:
                msg = " ".join([msg] + list(map(str, args)))

# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize

class ParserGenerator(object):
    def __init__(self, filename, stream=None):
        close_stream = None
        if stream is None:
            stream = open(filename)
            close_stream = stream.close
        self.filename = filename
        self.stream = stream
        self.generator = tokenize.generate_tokens(stream.readline)
        self.gettoken()  # Initialize lookahead
        self.dfas, self.startsymbol = self.parse()
        if close_stream is not None:
            close_stream()
        self.first = {}  # map from symbol name to set of tokens
        self.addfirstsets()

    def make_grammar(self):
        c = grammar.Grammar()
        names = list(self.dfas.keys())
        names.sort()
        names.remove(self.startsymbol)
        names.insert(0, self.startsymbol)
        for name in names:
            i = 256 + len(c.symbol2number)
            c.symbol2number[name] = i
            c.number2symbol[i] = name
        for name in names:
            dfa = self.dfas[name]
            states = []
            for state in dfa:
                arcs = []
                for label, next in state.arcs.items():
                    arcs.append((self.make_label(c, label), dfa.index(next)))
                if state.isfinal:
                    arcs.append((0, dfa.index(state)))
                states.append(arcs)
            c.states.append(states)
            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
        c.start = c.symbol2number[self.startsymbol]
        return c

    def make_first(self, c, name):
        rawfirst = self.first[name]
        first = {}
        for label in rawfirst:
            ilabel = self.make_label(c, label)
            ##assert ilabel not in first # XXX failed on <> ... !=
            first[ilabel] = 1
        return first

    def make_label(self, c, label):
        # XXX Maybe this should be a method on a subclass of converter?
        ilabel = len(c.labels)
        if label[0].isalpha():
            # Either a symbol name or a named token
            if label in c.symbol2number:
                # A symbol name (a non-terminal)
                if label in c.symbol2label:
                    return c.symbol2label[label]
                else:
                    c.labels.append((c.symbol2number[label], None))
                    c.symbol2label[label] = ilabel
                    return ilabel
            else:
                # A named token (NAME, NUMBER, STRING)
                itoken = getattr(token, label, None)
                assert isinstance(itoken, int), label
                assert itoken in token.tok_name, label
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel
        else:
            # Either a keyword or an operator
            assert label[0] in ('"', "'"), label
            value = eval(label)
            if value[0].isalpha():
                # A keyword
                if value in c.keywords:
                    return c.keywords[value]
                else:
                    c.labels.append((token.NAME, value))
                    c.keywords[value] = ilabel
                    return ilabel
            else:
                # An operator (any non-numeric token)
                itoken = token.opmap[value]  # Fails if unknown token
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel

    def addfirstsets(self):
        names = list(self.dfas.keys())
        names.sort()
        for name in names:
            if name not in self.first:
                self.calcfirst(name)

    def calcfirst(self, name):
        dfa = self.dfas[name]
        self.first[name] = None  # dummy to detect left recursion
        state = dfa[0]
        totalset = {}
        overlapcheck = {}
        for label, next in state.arcs.items():
            if label in self.dfas:
                if label in self.first:
                    fset = self.first[label]
                    if fset is None:
                        raise ValueError("recursion for rule %r" % name)
                else:
                    self.calcfirst(label)
                    fset = self.first[label]
                totalset.update(fset)
                overlapcheck[label] = fset
            else:
                totalset[label] = 1
                overlapcheck[label] = {label: 1}
        inverse = {}
        for label, itsfirst in overlapcheck.items():
            for symbol in itsfirst:
                if symbol in inverse:
                    raise ValueError("rule %s is ambiguous; %s is in the"
                                     " first sets of %s as well as %s" %
                                     (name, symbol, label, inverse[symbol]))
                inverse[symbol] = label
        self.first[name] = totalset

    def parse(self):
        dfas = {}
        startsymbol = None
        while self.type != token.ENDMARKER:
            while self.type == token.NEWLINE:
                self.gettoken()
            name = self.expect(token.NAME)
            self.expect(token.OP, ":")
            a, z = self.parse_rhs()
            self.expect(token.NEWLINE)
            dfa = self.make_dfa(a, z)
            self.simplify_dfa(dfa)
            dfas[name] = dfa
            #print name, oldlen, newlen
            if startsymbol is None:
                startsymbol = name
        return dfas, startsymbol

    def make_dfa(self, start, finish):
        # To turn an NFA into a DFA, we define the states of the DFA
        # to correspond to *sets* of states of the NFA.  Then do some
        # state reduction.  Let's represent sets as dicts with 1 for
        assert isinstance(start, NFAState)
        assert isinstance(finish, NFAState)

        def closure(state):
            base = {}
            addclosure(state, base)
            return base

        def addclosure(state, base):
            assert isinstance(state, NFAState)
            if state in base:
                return
            base[state] = 1
            for label, next in state.arcs:
                if label is None:
                    addclosure(next, base)

        states = [DFAState(closure(start), finish)]
        for state in states:  # NB states grows while we're iterating
            arcs = {}
            for nfastate in state.nfaset:
                for label, next in nfastate.arcs:
                    if label is not None:
                        addclosure(next, arcs.setdefault(label, {}))
            for label, nfaset in arcs.items():
                for st in states:
                    if st.nfaset == nfaset:
                        break
                else:
                    st = DFAState(nfaset, finish)
                    states.append(st)
                state.addarc(st, label)
        return states  # List of DFAState instances; first one is start

    def dump_nfa(self, name, start, finish):
        print("Dump of NFA for", name)
        todo = [start]
        for i, state in enumerate(todo):
            print("  State", i, state is finish and "(final)" or "")
            for label, next in state.arcs:
                if next in todo:
                    j = todo.index(next)
                else:
                    j = len(todo)
                    todo.append(next)
                if label is None:
                    print("    -> %d" % j)
                else:
                    print("    %s -> %d" % (label, j))

    def dump_dfa(self, name, dfa):
        print("Dump of DFA for", name)
        for i, state in enumerate(dfa):
            print("  State", i, state.isfinal and "(final)" or "")
            for label, next in state.arcs.items():
                print("    %s -> %d" % (label, dfa.index(next)))

    def simplify_dfa(self, dfa):
        # This is not theoretically optimal, but works well enough.
        # Algorithm: repeatedly look for two states that have the same
        # set of arcs (same labels pointing to the same nodes) and
        # unify them, until things stop changing.

        # dfa is a list of DFAState instances
        changes = True
        while changes:
            changes = False
            for i, state_i in enumerate(dfa):
                for j in range(i + 1, len(dfa)):
                    state_j = dfa[j]
                    if state_i == state_j:
                        del dfa[j]
                        for state in dfa:
                            state.unifystate(state_j, state_i)
                        changes = True
                        break

    def parse_rhs(self):
        a, z = self.parse_alt()
        if self.value != "|":
            return a, z
        else:
            aa = NFAState()
            zz = NFAState()
            aa.addarc(a)
            z.addarc(zz)
            while self.value == "|":
                self.gettoken()
                a, z = self.parse_alt()
                aa.addarc(a)
                z.addarc(zz)
            return aa, zz

    def parse_alt(self):
        a, b = self.parse_item()
        while (self.value in ("(", "[") or
               self.type in (token.NAME, token.STRING)):
            c, d = self.parse_item()
            b.addarc(c)
            b = d
        return a, b

    def parse_item(self):
        if self.value == "[":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, "]")
            a.addarc(z)
            return a, z
        else:
            a, z = self.parse_atom()
            value = self.value
            if value not in ("+", "*"):
                return a, z
            self.gettoken()
            z.addarc(a)
            if value == "+":
                return a, z
            else:
                return a, a

    def parse_atom(self):
        if self.value == "(":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, ")")
            return a, z
        elif self.type in (token.NAME, token.STRING):
            a = NFAState()
            z = NFAState()
            a.addarc(z, self.value)
            self.gettoken()
            return a, z
        else:
            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
                             self.type, self.value)

    def expect(self, type, value=None):
        if self.type != type or (value is not None and self.value != value):
            self.raise_error("expected %s/%s, got %s/%s",
                             type, value, self.type, self.value)
        value = self.value
        self.gettoken()
        return value

    def gettoken(self):
        tup = next(self.generator)
        while tup[0] in (token.COMMENT, token.NL):
            tup = next(self.generator)
        self.type, self.value, self.begin, prefix = tup

    def raise_error(self, msg, *args):
        if args:

        line = open(self.filename).readlines()[self.begin[0]]
        raise SyntaxError(msg, (self.filename, self.begin[0],
                                self.begin[1], line))

class NFAState(object):
    def __init__(self):
        self.arcs = []  # list of (label, NFAState) pairs

    def addarc(self, next, label=None):
        assert label is None or isinstance(label, str)
        assert isinstance(next, NFAState)
        self.arcs.append((label, next))

class DFAState(object):
    def __init__(self, nfaset, final):
        assert isinstance(nfaset, dict)
        assert isinstance(next(iter(nfaset)), NFAState)
        assert isinstance(final, NFAState)
        self.nfaset = nfaset
        self.isfinal = final in nfaset
        self.arcs = {}  # map from label to DFAState

    def addarc(self, next, label):
        assert isinstance(label, str)
        assert label not in self.arcs
        assert isinstance(next, DFAState)
        self.arcs[label] = next

    def unifystate(self, old, new):
        for label, next in self.arcs.items():
            if next is old:
                self.arcs[label] = new

    def __eq__(self, other):
        # Equality test -- ignore the nfaset instance variable
        assert isinstance(other, DFAState)
        if self.isfinal != other.isfinal:
            return False
        # Can't just return self.arcs == other.arcs, because that
        # would invoke this method recursively, with cycles...
        if len(self.arcs) != len(other.arcs):
            return False
        for label, next in self.arcs.items():
            if next is not other.arcs.get(label):
                return False
        return True

    __hash__ = None  # For Py3 compatibility.

def generate_grammar(filename="Grammar.txt"):
    p = ParserGenerator(filename)
    return p.make_grammar()


'''
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model
'''

# Author: Tarek Amr <@gr33ndata>

import sys, math

class UnseenTerms:

    def __init__(self):
        # Count unseen n-grams per each class
        self.doc_counts = {}
        # Count unseen n-grams per correct/incorrect results
        self.cic_counts = {}

    def _seen_unseel_label(self, seen, unseen):
        if seen + unseen:
            label = '%0.2f' % (float(unseen) / (seen+unseen))
        else:
            label = '%0.2f' %  0
        return label

    def per_cic(self, calculated_id='', actual_id='', seen_unseen=None):
        calculated_id = str(calculated_id).strip()
        actual_id =str(actual_id).strip()
        unseen_ratio_str = self._seen_unseel_label(*seen_unseen)
        if not unseen_ratio_str in self.cic_counts:
            self.cic_counts[unseen_ratio_str] = [0,0]
        if calculated_id == actual_id:
            self.cic_counts[unseen_ratio_str][0] += 1
        else:
            self.cic_counts[unseen_ratio_str][1] += 1

    def per_doc(self, doc_id='', seen_unseen=None):
        if not doc_id in self.doc_counts:
            self.doc_counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }
        self.doc_counts[doc_id]['seen'] += seen_unseen[0]
        self.doc_counts[doc_id]['unseen'] += seen_unseen[1]

    def display(self, per_doc=True, per_cic=True):
        if per_doc:
            print '\nUnseen n-grams as per actual doc_ids'
            doc_ids = self.doc_counts.keys()
            doc_ids.sort()
            for doc_id in doc_ids:
                unseen = self.doc_counts[doc_id]['unseen']
                total = self.doc_counts[doc_id]['unseen'] + self.doc_counts[doc_id]['seen']
                unseen_ratio =  float(unseen) / total
                print 'Ratio of unseen (%s): %0.4f' % (doc_id, unseen_ratio)
        if per_cic:
            print ''
            print 'Correct/Incorrect Unseen Ratios'
            cic_x = []
            correct = []
            incorrect = []
            cic_labels = self.cic_counts.keys()
            cic_labels.sort()
            for label in cic_labels:
                cic_x.append(str(label))
                correct.append(str(self.cic_counts[label][0]))
                incorrect.append(str(self.cic_counts[label][1]))
            print 'CIC X = [' + ','.join(cic_x) + ']'
            print 'Correct   = [' + ','.join(correct) + ']'
            print 'Incorrect = [' + ','.join(incorrect) + ']'

class LM:

    def __init__(self, n=3, lpad='', rpad='',
                 smoothing='Laplace', laplace_gama=1,
                 corpus_mix=0, corpus_mode='Miller',
                 verbose=False):
        '''
        Initialize our LM
        n: Order of ngram LM, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding.
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        smoothing: 'Laplace' or 'Witten'
        laplace_gama: Multiply 1 and V by this factor gamma
        corpus_mix: 0 (default) only use document probabilites
                  : or value between 0 and 1 lambda
                  : or c (or l) for Log-likelihood Odds model (Cooper LDA)
        corpus_mode: 'Hiemstra' or 'Miller'
                   : This tell us how to calculate pr_corpus(t)
                   : In fact, 'Hiemstra' makes sense only with Multivaria LM
                   : Whereas the LM here is Multinomial.
        '''
        if n < 1:
            raise ValueError('n should be >= 1')
        self.n = n
        # Counters for joint probabilities
        self.term_count_n = {}
        self.term_count_n_1 = {}
        # To be used in case of mixing doc prob with corpus prob.
        self.corpus_count_n = {'ngrams': {}, 'total': 0}
        self.corpus_count_n_1 = {'ngrams': {}, 'total': 0}
        self.doc_lengths = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.smoothing = smoothing
        self.laplace_gama = float(laplace_gama)
        if not type(corpus_mix) is str:
            self.corpus_mix = min(float(corpus_mix),1)
        else:
            self.corpus_mix = corpus_mix
        self.corpus_mode = corpus_mode
        self.joiner = ''
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def get_vocabulary(self):
        return self.vocabulary

    def display(self, per_doc=True, per_cic=True):
        '''
        Displays statistics about our LM
        '''
        voc_list = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            ngrams = len(self.term_count_n[doc_id]['ngrams'])
            print 'n-Grams (doc %s): %d' % (str(doc_id), ngrams)
            ngrams1 = len(self.term_count_n_1[doc_id]['ngrams'])
            print '(n-1)-Grams (doc %s): %d' % (str(doc_id), ngrams1)
            voc_list.append(ngrams)
        print 'Classed Vocabularies:', voc_list
        print ''
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        print 'n-Grams (collection): %d' % (corpus_ngrams)
        corpus_ngrams1 = len(self.corpus_count_n_1['ngrams'])
        print '(n-1)-Grams (collection): %d' % (corpus_ngrams1)
        self.unseen_counts.display(per_doc, per_cic)
        # Display overlapping n-grams between classes

    def get_ngram_counts(self):
        ''' Returns a list of n-gram counts
            Array of classes counts and last item is for corpus
        '''
        ngram_counts = {
            'classes': [],
            'classes-1': [],
            'corpus': 0,
            'corpus-1': 0
        }
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            #print self.term_count_n[doc_id]
            class_ngrams = len(self.term_count_n[doc_id]['ngrams'])
            class_n1grams = len(self.term_count_n_1[doc_id]['ngrams'])
            ngram_counts['classes'].append(class_ngrams)
            ngram_counts['classes-1'].append(class_n1grams)
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        corpus_n1grams = len(self.corpus_count_n_1['ngrams'])
        ngram_counts['corpus'] = corpus_ngrams
        ngram_counts['corpus-1'] = corpus_n1grams
        return ngram_counts

    def overlaps(self):
        ''' Print overlapping n-grams between classes.
        '''
        omx = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            row = [0] * len(doc_ids)
            omx.append(row)

        for i, doc_id_i in enumerate(doc_ids):

            ngrams = len(self.term_count_n[doc_id_i]['ngrams'])
            omx[i][i] = ngrams
            for j in range(i):
                doc_id_j = doc_ids[j]
                ongrams = 0
                for ngram in self.term_count_n[doc_id_i]['ngrams']:
                    if ngram in self.term_count_n[doc_id_j]['ngrams']:
                        ongrams += 1
                    omx[i][j] = 0 #ongrams
                    omx[j][i] = ongrams
        print '\nn-gram overlaps:'
        print doc_ids
        for i in range(len(omx)):
            row = []
            for j in range(len(omx[i])):
                row.append( round( float(omx[i][j] * 100) / omx[i][i],2 ) )
            print row

    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams
            terms: list of terms
        '''
        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams

'''
        Pad doc from the left and right before adding,
        depending on what's in self.lpad and self.rpad
        If any of them is '', then don't pad there.
        '''
        lpad = rpad = []
        if self.lpad:
            lpad = [self.lpad] * (self.n - 1)
        if self.rpad:
            rpad = [self.rpad] * (self.n - 1)
        return lpad + terms + rpad

    def update_counts(self, doc_id ='', ngrams=[]):
        if not doc_id in self.term_count_n:
            self.term_count_n[doc_id] = {'ngrams': {}, 'total': 0}
        if not doc_id in self.term_count_n_1:
            self.term_count_n_1[doc_id] = {'ngrams': {}, 'total': 0}
        for ngram in ngrams:
            # Generate n-grams and sub-ngrams
            ngram_n = self.joiner.join(ngram)
            ngram_n_1 = self.joiner.join(ngram[:-1])
            # Update n-gram counts for doc_id
            if ngram_n in self.term_count_n[doc_id]['ngrams']:
                self.term_count_n[doc_id]['ngrams'][ngram_n] += 1
            else:
                self.term_count_n[doc_id]['ngrams'][ngram_n] = 1
            self.term_count_n[doc_id]['total'] += 1
            # Update n-gram counts for corpus
            if ngram_n in self.corpus_count_n['ngrams']:
                self.corpus_count_n['ngrams'][ngram_n] += 1
            else:
                self.corpus_count_n['ngrams'][ngram_n] = 1
            self.corpus_count_n['total'] += 1
            if ngram_n_1 in self.term_count_n_1[doc_id]['ngrams']:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] += 1
            else:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] = 1
            self.term_count_n_1[doc_id]['total'] += 1
            if ngram_n_1 in self.corpus_count_n_1['ngrams']:
                self.corpus_count_n_1['ngrams'][ngram_n_1] += 1
            else:
                self.corpus_count_n_1['ngrams'][ngram_n_1] = 1
            self.corpus_count_n_1['total'] += 1

    def update_lengths(self, doc_id ='', doc_length=0):
        if not doc_id in self.doc_lengths:
            self.doc_lengths[doc_id] = {'lengths': [], 'mean': -1}
        self.doc_lengths[doc_id]['lengths'].append(int(doc_length))

    def get_mean_lengths(self, doc_id =''):
        my_mean_len = 0
        others_mean_len = []
        for d_id in self.doc_lengths:
            if self.doc_lengths[d_id]['mean'] == -1:
                dlen = self.doc_lengths[d_id]['lengths']
                doc_mean_len = float(sum(dlen)) / len(dlen)
                self.doc_lengths[d_id]['mean'] = doc_mean_len
            else:
                doc_mean_len = self.doc_lengths[d_id]['mean']
            if d_id == doc_id:
                my_mean_len = doc_mean_len
            else:
                others_mean_len.append(doc_mean_len)
        oth_mean_len = float(sum(others_mean_len)) / len(others_mean_len)
        return (my_mean_len, oth_mean_len)

    def add_doc(self, doc_id ='', doc_terms=[], doc_length=-1):
        '''
        Add new document to our Language Model (training phase)
        doc_id is used here, so we build seperate LF for each doc_id
        I.e. if you call it more than once with same doc_id,
        then all terms given via doc_terms will contribute to same LM
        doc_terms: list of words in document to be added
        doc_length: the length of the document, you can provide it yourself,
                    otherwise, we use len(doc_terms) instead.
        '''
        if doc_length == -1:
            self.update_lengths(doc_id=doc_id, doc_length=len(doc_terms))
        else:
            self.update_lengths(doc_id=doc_id, doc_length=int(doc_length))
        for term in doc_terms:
            self.vocabulary.add(term)
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        self.update_counts(doc_id, ngrams)

    def laplace(self, x, y, doc_id):
        ''' Laplace (Add One/Gamma) Smoothing
        '''
        add_numer = 1 * self.laplace_gama
        laplace_mode = 'n-1'
        if laplace_mode == 'ch':
            v = len(self.vocabulary)
        elif laplace_mode == 'ch^n-1':
            v = math.pow(len(self.vocabulary),self.n-1)
        else:
            if doc_id:
                v = len(self.term_count_n_1[doc_id]['ngrams'])
            else:
                v = len(self.corpus_count_n_1['ngrams'])
        add_denom = v * self.laplace_gama
        return float(x + add_numer) / float(y + add_denom)

    def witten(self, count, n, t, log, new_doc):
        ''' Witten-Bell Smoothing
        '''
        self.return_unseen = True if new_doc else False
        if count:
            return float(count) / (n+t)
        elif self.return_unseen:
            return float(t) / (n+t)
        elif log:
            return 1
        else:
            return 1

    def pr_ngram(self, doc_id, ngram, new_doc=False, log=True, logbase=2, doc_length=-1):
        ngram_n = self.joiner.join(ngram)
        ngram_n_1 = self.joiner.join(ngram[:-1])
        ngram_n_count = self.term_count_n[doc_id]['ngrams'].get(ngram_n,0)
        ngram_n_1_count = self.term_count_n_1[doc_id]['ngrams'].get(ngram_n_1,0)
        # Apply smoothing
        if self.smoothing == 'Laplace':
            pr = self.laplace(ngram_n_count, ngram_n_1_count, doc_id)
            if self.corpus_mix == 'c' or self.corpus_mix == 'l':
                corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
                corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
                pr_dash = self.laplace(corpus_ngram_n_count - ngram_n_count,
                                  corpus_ngram_n_1_count - ngram_n_1_count,
                                  doc_id='')
                pr_mix = float(pr) / float(pr_dash)
            elif type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr
        elif self.smoothing == 'Witten':
            wittenn = ngram_n_1_count
            wittent = len([key for key in self.term_count_n[doc_id]['ngrams'] if key.startswith(ngram_n_1)])
            pr = self.witten(ngram_n_count, wittenn, wittent, log, new_doc)
            pr_mix = pr
        else:
            pr = float(ngram_n_count) / float(ngram_n_1_count)
            if type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr

        # Update seen/unseen counts
        if ngram_n_count:
            seen = True
        else:
            seen = False
        if log:
            return (-math.log(pr_mix,logbase),seen)
        else:
            return (pr_mix,seen)

    def pr_corpus(self, ngram_n, ngram_n_1):
        if self.corpus_mode == 'Hiemstra':
            df = 0
            df_total = 0
            for doc_id in self.term_count_n_1:
                if self.term_count_n[doc_id]['ngrams'].get(ngram_n,0):
                    df += 1
                df_total += 1
            pr = float(df) / float(df_total)
            if not pr:
                pr += 0.0001
        else:
            corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
            corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
            pr = self.laplace(corpus_ngram_n_count,
                              corpus_ngram_n_1_count,
                              doc_id='')
        return pr

    def pr_doc(self, doc_id, log=True, logbase=2, doc_length=-1):
        ''' This method may be overridden by implementers
            Here we assume uniform Pr(doc) within Bayes rule
            i.e. Pr(doc/string) = Pr(string/doc)
            rather than Pr(doc/string) = Pr(string/doc) * Pr(doc)
        '''
        if log:
            return 0
        else:
            return 1

    def calculate(self, doc_terms=[], actual_id='', doc_length=-1):
        '''
        Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }
        '''
        calculated = {
            'prob': -1,
            'calc_id': '',
            'actual_id': actual_id,
            'seen_unseen_count': (0,0)
        }
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        for doc_id in self.term_count_n:
            doc_pr = 0
            new_doc = True
            seen_count = 0
            unseen_count = 0
            for ngram in ngrams:
                (ngram_pr, ngram_seen) = self.pr_ngram(doc_id, ngram,
                        new_doc=new_doc, log=True, logbase=2, doc_length=doc_length)
                doc_pr += ngram_pr
                new_doc = False
                if ngram_seen:
                    seen_count += 1
                else:
                    unseen_count += 1
            doc_pr += self.pr_doc(doc_id, doc_length=doc_length)
            if self.verbose:
                print doc_id, actual_id, doc_pr
            if calculated['prob'] == -1 or doc_pr < calculated['prob']:
                calculated['prob'] = doc_pr
                calculated['calc_id'] = doc_id
                calculated['seen_unseen_count'] = (seen_count, unseen_count)
        self.unseen_counts.per_doc(doc_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        self.unseen_counts.per_cic(calculated_id=calculated['calc_id'],
            actual_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        return calculated

if __name__ == '__main__':

    #Preprocessor
    def term2ch(text):
        return [ch for ch in text]

    lm = LM(n=2, verbose=True, smoothing='Laplace', corpus_mix=0)

    #print lm.to_ngrams(term2ch('hello dear world'))

    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i see glass of apple juice'))
    lm.add_doc(doc_id='apple', doc_terms=term2ch('the tree is full or apples'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i see the orange tree shaking'))
    lm.add_doc(doc_id='orange', doc_terms=term2ch('orange orange juice'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i do not like jaffa cake'))
    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i have apple juice'))
    #print lm.pr
    result = lm.calculate(doc_terms=term2ch('orango juice'))
    print result

    print lm.term_count_n
    print lm.term_count_n_1
    print lm.vocabulary
    print lm.overlaps()


if seen + unseen:
            label = '%0.2f' % (float(unseen) / (seen+unseen))
        else:
            label = '%0.2f' %  0
        return label

'''
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model
'''

# Author: Tarek Amr <@gr33ndata>

import sys, math

class UnseenTerms:

    def __init__(self):
        # Count unseen n-grams per each class
        self.doc_counts = {}
        # Count unseen n-grams per correct/incorrect results
        self.cic_counts = {}

    def _seen_unseel_label(self, seen, unseen):

    def per_cic(self, calculated_id='', actual_id='', seen_unseen=None):
        calculated_id = str(calculated_id).strip()
        actual_id =str(actual_id).strip()
        unseen_ratio_str = self._seen_unseel_label(*seen_unseen)
        if not unseen_ratio_str in self.cic_counts:
            self.cic_counts[unseen_ratio_str] = [0,0]
        if calculated_id == actual_id:
            self.cic_counts[unseen_ratio_str][0] += 1
        else:
            self.cic_counts[unseen_ratio_str][1] += 1

    def per_doc(self, doc_id='', seen_unseen=None):
        if not doc_id in self.doc_counts:
            self.doc_counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }
        self.doc_counts[doc_id]['seen'] += seen_unseen[0]
        self.doc_counts[doc_id]['unseen'] += seen_unseen[1]

    def display(self, per_doc=True, per_cic=True):
        if per_doc:
            print '\nUnseen n-grams as per actual doc_ids'
            doc_ids = self.doc_counts.keys()
            doc_ids.sort()
            for doc_id in doc_ids:
                unseen = self.doc_counts[doc_id]['unseen']
                total = self.doc_counts[doc_id]['unseen'] + self.doc_counts[doc_id]['seen']
                unseen_ratio =  float(unseen) / total
                print 'Ratio of unseen (%s): %0.4f' % (doc_id, unseen_ratio)
        if per_cic:
            print ''
            print 'Correct/Incorrect Unseen Ratios'
            cic_x = []
            correct = []
            incorrect = []
            cic_labels = self.cic_counts.keys()
            cic_labels.sort()
            for label in cic_labels:
                cic_x.append(str(label))
                correct.append(str(self.cic_counts[label][0]))
                incorrect.append(str(self.cic_counts[label][1]))
            print 'CIC X = [' + ','.join(cic_x) + ']'
            print 'Correct   = [' + ','.join(correct) + ']'
            print 'Incorrect = [' + ','.join(incorrect) + ']'

class LM:

    def __init__(self, n=3, lpad='', rpad='',
                 smoothing='Laplace', laplace_gama=1,
                 corpus_mix=0, corpus_mode='Miller',
                 verbose=False):
        '''
        Initialize our LM
        n: Order of ngram LM, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding.
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        smoothing: 'Laplace' or 'Witten'
        laplace_gama: Multiply 1 and V by this factor gamma
        corpus_mix: 0 (default) only use document probabilites
                  : or value between 0 and 1 lambda
                  : or c (or l) for Log-likelihood Odds model (Cooper LDA)
        corpus_mode: 'Hiemstra' or 'Miller'
                   : This tell us how to calculate pr_corpus(t)
                   : In fact, 'Hiemstra' makes sense only with Multivaria LM
                   : Whereas the LM here is Multinomial.
        '''
        if n < 1:
            raise ValueError('n should be >= 1')
        self.n = n
        # Counters for joint probabilities
        self.term_count_n = {}
        self.term_count_n_1 = {}
        # To be used in case of mixing doc prob with corpus prob.
        self.corpus_count_n = {'ngrams': {}, 'total': 0}
        self.corpus_count_n_1 = {'ngrams': {}, 'total': 0}
        self.doc_lengths = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.smoothing = smoothing
        self.laplace_gama = float(laplace_gama)
        if not type(corpus_mix) is str:
            self.corpus_mix = min(float(corpus_mix),1)
        else:
            self.corpus_mix = corpus_mix
        self.corpus_mode = corpus_mode
        self.joiner = ''
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def get_vocabulary(self):
        return self.vocabulary

    def display(self, per_doc=True, per_cic=True):
        '''
        Displays statistics about our LM
        '''
        voc_list = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            ngrams = len(self.term_count_n[doc_id]['ngrams'])
            print 'n-Grams (doc %s): %d' % (str(doc_id), ngrams)
            ngrams1 = len(self.term_count_n_1[doc_id]['ngrams'])
            print '(n-1)-Grams (doc %s): %d' % (str(doc_id), ngrams1)
            voc_list.append(ngrams)
        print 'Classed Vocabularies:', voc_list
        print ''
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        print 'n-Grams (collection): %d' % (corpus_ngrams)
        corpus_ngrams1 = len(self.corpus_count_n_1['ngrams'])
        print '(n-1)-Grams (collection): %d' % (corpus_ngrams1)
        self.unseen_counts.display(per_doc, per_cic)
        # Display overlapping n-grams between classes

    def get_ngram_counts(self):
        ''' Returns a list of n-gram counts
            Array of classes counts and last item is for corpus
        '''
        ngram_counts = {
            'classes': [],
            'classes-1': [],
            'corpus': 0,
            'corpus-1': 0
        }
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            #print self.term_count_n[doc_id]
            class_ngrams = len(self.term_count_n[doc_id]['ngrams'])
            class_n1grams = len(self.term_count_n_1[doc_id]['ngrams'])
            ngram_counts['classes'].append(class_ngrams)
            ngram_counts['classes-1'].append(class_n1grams)
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        corpus_n1grams = len(self.corpus_count_n_1['ngrams'])
        ngram_counts['corpus'] = corpus_ngrams
        ngram_counts['corpus-1'] = corpus_n1grams
        return ngram_counts

    def overlaps(self):
        ''' Print overlapping n-grams between classes.
        '''
        omx = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            row = [0] * len(doc_ids)
            omx.append(row)

        for i, doc_id_i in enumerate(doc_ids):

            ngrams = len(self.term_count_n[doc_id_i]['ngrams'])
            omx[i][i] = ngrams
            for j in range(i):
                doc_id_j = doc_ids[j]
                ongrams = 0
                for ngram in self.term_count_n[doc_id_i]['ngrams']:
                    if ngram in self.term_count_n[doc_id_j]['ngrams']:
                        ongrams += 1
                    omx[i][j] = 0 #ongrams
                    omx[j][i] = ongrams
        print '\nn-gram overlaps:'
        print doc_ids
        for i in range(len(omx)):
            row = []
            for j in range(len(omx[i])):
                row.append( round( float(omx[i][j] * 100) / omx[i][i],2 ) )
            print row

    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams
            terms: list of terms
        '''
        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams

    def lr_padding(self, terms):
        '''
        Pad doc from the left and right before adding,
        depending on what's in self.lpad and self.rpad
        If any of them is '', then don't pad there.
        '''
        lpad = rpad = []
        if self.lpad:
            lpad = [self.lpad] * (self.n - 1)
        if self.rpad:
            rpad = [self.rpad] * (self.n - 1)
        return lpad + terms + rpad

    def update_counts(self, doc_id ='', ngrams=[]):
        if not doc_id in self.term_count_n:
            self.term_count_n[doc_id] = {'ngrams': {}, 'total': 0}
        if not doc_id in self.term_count_n_1:
            self.term_count_n_1[doc_id] = {'ngrams': {}, 'total': 0}
        for ngram in ngrams:
            # Generate n-grams and sub-ngrams
            ngram_n = self.joiner.join(ngram)
            ngram_n_1 = self.joiner.join(ngram[:-1])
            # Update n-gram counts for doc_id
            if ngram_n in self.term_count_n[doc_id]['ngrams']:
                self.term_count_n[doc_id]['ngrams'][ngram_n] += 1
            else:
                self.term_count_n[doc_id]['ngrams'][ngram_n] = 1
            self.term_count_n[doc_id]['total'] += 1
            # Update n-gram counts for corpus
            if ngram_n in self.corpus_count_n['ngrams']:
                self.corpus_count_n['ngrams'][ngram_n] += 1
            else:
                self.corpus_count_n['ngrams'][ngram_n] = 1
            self.corpus_count_n['total'] += 1
            if ngram_n_1 in self.term_count_n_1[doc_id]['ngrams']:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] += 1
            else:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] = 1
            self.term_count_n_1[doc_id]['total'] += 1
            if ngram_n_1 in self.corpus_count_n_1['ngrams']:
                self.corpus_count_n_1['ngrams'][ngram_n_1] += 1
            else:
                self.corpus_count_n_1['ngrams'][ngram_n_1] = 1
            self.corpus_count_n_1['total'] += 1

    def update_lengths(self, doc_id ='', doc_length=0):
        if not doc_id in self.doc_lengths:
            self.doc_lengths[doc_id] = {'lengths': [], 'mean': -1}
        self.doc_lengths[doc_id]['lengths'].append(int(doc_length))

    def get_mean_lengths(self, doc_id =''):
        my_mean_len = 0
        others_mean_len = []
        for d_id in self.doc_lengths:
            if self.doc_lengths[d_id]['mean'] == -1:
                dlen = self.doc_lengths[d_id]['lengths']
                doc_mean_len = float(sum(dlen)) / len(dlen)
                self.doc_lengths[d_id]['mean'] = doc_mean_len
            else:
                doc_mean_len = self.doc_lengths[d_id]['mean']
            if d_id == doc_id:
                my_mean_len = doc_mean_len
            else:
                others_mean_len.append(doc_mean_len)
        oth_mean_len = float(sum(others_mean_len)) / len(others_mean_len)
        return (my_mean_len, oth_mean_len)

    def add_doc(self, doc_id ='', doc_terms=[], doc_length=-1):
        '''
        Add new document to our Language Model (training phase)
        doc_id is used here, so we build seperate LF for each doc_id
        I.e. if you call it more than once with same doc_id,
        then all terms given via doc_terms will contribute to same LM
        doc_terms: list of words in document to be added
        doc_length: the length of the document, you can provide it yourself,
                    otherwise, we use len(doc_terms) instead.
        '''
        if doc_length == -1:
            self.update_lengths(doc_id=doc_id, doc_length=len(doc_terms))
        else:
            self.update_lengths(doc_id=doc_id, doc_length=int(doc_length))
        for term in doc_terms:
            self.vocabulary.add(term)
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        self.update_counts(doc_id, ngrams)

    def laplace(self, x, y, doc_id):
        ''' Laplace (Add One/Gamma) Smoothing
        '''
        add_numer = 1 * self.laplace_gama
        laplace_mode = 'n-1'
        if laplace_mode == 'ch':
            v = len(self.vocabulary)
        elif laplace_mode == 'ch^n-1':
            v = math.pow(len(self.vocabulary),self.n-1)
        else:
            if doc_id:
                v = len(self.term_count_n_1[doc_id]['ngrams'])
            else:
                v = len(self.corpus_count_n_1['ngrams'])
        add_denom = v * self.laplace_gama
        return float(x + add_numer) / float(y + add_denom)

    def witten(self, count, n, t, log, new_doc):
        ''' Witten-Bell Smoothing
        '''
        self.return_unseen = True if new_doc else False
        if count:
            return float(count) / (n+t)
        elif self.return_unseen:
            return float(t) / (n+t)
        elif log:
            return 1
        else:
            return 1

    def pr_ngram(self, doc_id, ngram, new_doc=False, log=True, logbase=2, doc_length=-1):
        ngram_n = self.joiner.join(ngram)
        ngram_n_1 = self.joiner.join(ngram[:-1])
        ngram_n_count = self.term_count_n[doc_id]['ngrams'].get(ngram_n,0)
        ngram_n_1_count = self.term_count_n_1[doc_id]['ngrams'].get(ngram_n_1,0)
        # Apply smoothing
        if self.smoothing == 'Laplace':
            pr = self.laplace(ngram_n_count, ngram_n_1_count, doc_id)
            if self.corpus_mix == 'c' or self.corpus_mix == 'l':
                corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
                corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
                pr_dash = self.laplace(corpus_ngram_n_count - ngram_n_count,
                                  corpus_ngram_n_1_count - ngram_n_1_count,
                                  doc_id='')
                pr_mix = float(pr) / float(pr_dash)
            elif type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr
        elif self.smoothing == 'Witten':
            wittenn = ngram_n_1_count
            wittent = len([key for key in self.term_count_n[doc_id]['ngrams'] if key.startswith(ngram_n_1)])
            pr = self.witten(ngram_n_count, wittenn, wittent, log, new_doc)
            pr_mix = pr
        else:
            pr = float(ngram_n_count) / float(ngram_n_1_count)
            if type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr

        # Update seen/unseen counts
        if ngram_n_count:
            seen = True
        else:
            seen = False
        if log:
            return (-math.log(pr_mix,logbase),seen)
        else:
            return (pr_mix,seen)

    def pr_corpus(self, ngram_n, ngram_n_1):
        if self.corpus_mode == 'Hiemstra':
            df = 0
            df_total = 0
            for doc_id in self.term_count_n_1:
                if self.term_count_n[doc_id]['ngrams'].get(ngram_n,0):
                    df += 1
                df_total += 1
            pr = float(df) / float(df_total)
            if not pr:
                pr += 0.0001
        else:
            corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
            corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
            pr = self.laplace(corpus_ngram_n_count,
                              corpus_ngram_n_1_count,
                              doc_id='')
        return pr

    def pr_doc(self, doc_id, log=True, logbase=2, doc_length=-1):
        ''' This method may be overridden by implementers
            Here we assume uniform Pr(doc) within Bayes rule
            i.e. Pr(doc/string) = Pr(string/doc)
            rather than Pr(doc/string) = Pr(string/doc) * Pr(doc)
        '''
        if log:
            return 0
        else:
            return 1

    def calculate(self, doc_terms=[], actual_id='', doc_length=-1):
        '''
        Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }
        '''
        calculated = {
            'prob': -1,
            'calc_id': '',
            'actual_id': actual_id,
            'seen_unseen_count': (0,0)
        }
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        for doc_id in self.term_count_n:
            doc_pr = 0
            new_doc = True
            seen_count = 0
            unseen_count = 0
            for ngram in ngrams:
                (ngram_pr, ngram_seen) = self.pr_ngram(doc_id, ngram,
                        new_doc=new_doc, log=True, logbase=2, doc_length=doc_length)
                doc_pr += ngram_pr
                new_doc = False
                if ngram_seen:
                    seen_count += 1
                else:
                    unseen_count += 1
            doc_pr += self.pr_doc(doc_id, doc_length=doc_length)
            if self.verbose:
                print doc_id, actual_id, doc_pr
            if calculated['prob'] == -1 or doc_pr < calculated['prob']:
                calculated['prob'] = doc_pr
                calculated['calc_id'] = doc_id
                calculated['seen_unseen_count'] = (seen_count, unseen_count)
        self.unseen_counts.per_doc(doc_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        self.unseen_counts.per_cic(calculated_id=calculated['calc_id'],
            actual_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        return calculated

if __name__ == '__main__':

    #Preprocessor
    def term2ch(text):
        return [ch for ch in text]

    lm = LM(n=2, verbose=True, smoothing='Laplace', corpus_mix=0)

    #print lm.to_ngrams(term2ch('hello dear world'))

    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i see glass of apple juice'))
    lm.add_doc(doc_id='apple', doc_terms=term2ch('the tree is full or apples'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i see the orange tree shaking'))
    lm.add_doc(doc_id='orange', doc_terms=term2ch('orange orange juice'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i do not like jaffa cake'))
    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i have apple juice'))
    #print lm.pr
    result = lm.calculate(doc_terms=term2ch('orango juice'))
    print result

    print lm.term_count_n
    print lm.term_count_n_1
    print lm.vocabulary
    print lm.overlaps()


'''
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model
'''

# Author: Tarek Amr <@gr33ndata>

import sys, math

class UnseenTerms:

    def __init__(self):
        # Count unseen n-grams per each class
        self.doc_counts = {}
        # Count unseen n-grams per correct/incorrect results
        self.cic_counts = {}

    def _seen_unseel_label(self, seen, unseen):
        if seen + unseen:
            label = '%0.2f' % (float(unseen) / (seen+unseen))
        else:
            label = '%0.2f' %  0
        return label

    def per_cic(self, calculated_id='', actual_id='', seen_unseen=None):
        calculated_id = str(calculated_id).strip()
        actual_id =str(actual_id).strip()
        unseen_ratio_str = self._seen_unsee

        if not unseen_ratio_str in self.cic_counts:
            self.cic_counts[unseen_ratio_str] = [0,0]
        if calculated_id == actual_id:
            self.cic_counts[unseen_ratio_str][0] += 1
        else:
            self.cic_counts[unseen_ratio_str][1] += 1

    def per_doc(self, doc_id='', seen_unseen=None):
        if not doc_id in self.doc_counts:
            self.doc_counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }
        self.doc_counts[doc_id]['seen'] += seen_unseen[0]
        self.doc_counts[doc_id]['unseen'] += seen_unseen[1]

    def display(self, per_doc=True, per_cic=True):
        if per_doc:
            print '\nUnseen n-grams as per actual doc_ids'
            doc_ids = self.doc_counts.keys()
            doc_ids.sort()
            for doc_id in doc_ids:
                unseen = self.doc_counts[doc_id]['unseen']
                total = self.doc_counts[doc_id]['unseen'] + self.doc_counts[doc_id]['seen']
                unseen_ratio =  float(unseen) / total
                print 'Ratio of unseen (%s): %0.4f' % (doc_id, unseen_ratio)
        if per_cic:
            print ''
            print 'Correct/Incorrect Unseen Ratios'
            cic_x = []
            correct = []
            incorrect = []
            cic_labels = self.cic_counts.keys()
            cic_labels.sort()
            for label in cic_labels:
                cic_x.append(str(label))
                correct.append(str(self.cic_counts[label][0]))
                incorrect.append(str(self.cic_counts[label][1]))
            print 'CIC X = [' + ','.join(cic_x) + ']'
            print 'Correct   = [' + ','.join(correct) + ']'
            print 'Incorrect = [' + ','.join(incorrect) + ']'

class LM:

    def __init__(self, n=3, lpad='', rpad='',
                 smoothing='Laplace', laplace_gama=1,
                 corpus_mix=0, corpus_mode='Miller',
                 verbose=False):
        '''
        Initialize our LM
        n: Order of ngram LM, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding.
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        smoothing: 'Laplace' or 'Witten'
        laplace_gama: Multiply 1 and V by this factor gamma
        corpus_mix: 0 (default) only use document probabilites
                  : or value between 0 and 1 lambda
                  : or c (or l) for Log-likelihood Odds model (Cooper LDA)
        corpus_mode: 'Hiemstra' or 'Miller'
                   : This tell us how to calculate pr_corpus(t)
                   : In fact, 'Hiemstra' makes sense only with Multivaria LM
                   : Whereas the LM here is Multinomial.
        '''
        if n < 1:
            raise ValueError('n should be >= 1')
        self.n = n
        # Counters for joint probabilities
        self.term_count_n = {}
        self.term_count_n_1 = {}
        # To be used in case of mixing doc prob with corpus prob.
        self.corpus_count_n = {'ngrams': {}, 'total': 0}
        self.corpus_count_n_1 = {'ngrams': {}, 'total': 0}
        self.doc_lengths = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.smoothing = smoothing
        self.laplace_gama = float(laplace_gama)
        if not type(corpus_mix) is str:
            self.corpus_mix = min(float(corpus_mix),1)
        else:
            self.corpus_mix = corpus_mix
        self.corpus_mode = corpus_mode
        self.joiner = ''
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def get_vocabulary(self):
        return self.vocabulary

    def display(self, per_doc=True, per_cic=True):
        '''
        Displays statistics about our LM
        '''
        voc_list = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            ngrams = len(self.term_count_n[doc_id]['ngrams'])
            print 'n-Grams (doc %s): %d' % (str(doc_id), ngrams)
            ngrams1 = len(self.term_count_n_1[doc_id]['ngrams'])
            print '(n-1)-Grams (doc %s): %d' % (str(doc_id), ngrams1)
            voc_list.append(ngrams)
        print 'Classed Vocabularies:', voc_list
        print ''
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        print 'n-Grams (collection): %d' % (corpus_ngrams)
        corpus_ngrams1 = len(self.corpus_count_n_1['ngrams'])
        print '(n-1)-Grams (collection): %d' % (corpus_ngrams1)
        self.unseen_counts.display(per_doc, per_cic)
        # Display overlapping n-grams between classes

    def get_ngram_counts(self):
        ''' Returns a list of n-gram counts
            Array of classes counts and last item is for corpus
        '''
        ngram_counts = {
            'classes': [],
            'classes-1': [],
            'corpus': 0,
            'corpus-1': 0
        }
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            #print self.term_count_n[doc_id]
            class_ngrams = len(self.term_count_n[doc_id]['ngrams'])
            class_n1grams = len(self.term_count_n_1[doc_id]['ngrams'])
            ngram_counts['classes'].append(class_ngrams)
            ngram_counts['classes-1'].append(class_n1grams)
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        corpus_n1grams = len(self.corpus_count_n_1['ngrams'])
        ngram_counts['corpus'] = corpus_ngrams
        ngram_counts['corpus-1'] = corpus_n1grams
        return ngram_counts

    def overlaps(self):
        ''' Print overlapping n-grams between classes.
        '''
        omx = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            row = [0] * len(doc_ids)
            omx.append(row)

        for i, doc_id_i in enumerate(doc_ids):

            ngrams = len(self.term_count_n[doc_id_i]['ngrams'])
            omx[i][i] = ngrams
            for j in range(i):
                doc_id_j = doc_ids[j]
                ongrams = 0
                for ngram in self.term_count_n[doc_id_i]['ngrams']:
                    if ngram in self.term_count_n[doc_id_j]['ngrams']:
                        ongrams += 1
                    omx[i][j] = 0 #ongrams
                    omx[j][i] = ongrams
        print '\nn-gram overlaps:'
        print doc_ids
        for i in range(len(omx)):
            row = []
            for j in range(len(omx[i])):
                row.append( round( float(omx[i][j] * 100) / omx[i][i],2 ) )
            print row

    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams
            terms: list of terms
        '''
        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams

    def lr_padding(self, terms):
        '''
        Pad doc from the left and right before adding,
        depending on what's in self.lpad and self.rpad
        If any of them is '', then don't pad there.
        '''
        lpad = rpad = []
        if self.lpad:
            lpad = [self.lpad] * (self.n - 1)
        if self.rpad:
            rpad = [self.rpad] * (self.n - 1)
        return lpad + terms + rpad

    def update_counts(self, doc_id ='', ngrams=[]):
        if not doc_id in self.term_count_n:
            self.term_count_n[doc_id] = {'ngrams': {}, 'total': 0}
        if not doc_id in self.term_count_n_1:
            self.term_count_n_1[doc_id] = {'ngrams': {}, 'total': 0}
        for ngram in ngrams:
            # Generate n-grams and sub-ngrams
            ngram_n = self.joiner.join(ngram)
            ngram_n_1 = self.joiner.join(ngram[:-1])
            # Update n-gram counts for doc_id
            if ngram_n in self.term_count_n[doc_id]['ngrams']:
                self.term_count_n[doc_id]['ngrams'][ngram_n] += 1
            else:
                self.term_count_n[doc_id]['ngrams'][ngram_n] = 1
            self.term_count_n[doc_id]['total'] += 1
            # Update n-gram counts for corpus
            if ngram_n in self.corpus_count_n['ngrams']:
                self.corpus_count_n['ngrams'][ngram_n] += 1
            else:
                self.corpus_count_n['ngrams'][ngram_n] = 1
            self.corpus_count_n['total'] += 1
            if ngram_n_1 in self.term_count_n_1[doc_id]['ngrams']:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] += 1
            else:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] = 1
            self.term_count_n_1[doc_id]['total'] += 1
            if ngram_n_1 in self.corpus_count_n_1['ngrams']:
                self.corpus_count_n_1['ngrams'][ngram_n_1] += 1
            else:
                self.corpus_count_n_1['ngrams'][ngram_n_1] = 1
            self.corpus_count_n_1['total'] += 1

    def update_lengths(self, doc_id ='', doc_length=0):
        if not doc_id in self.doc_lengths:
            self.doc_lengths[doc_id] = {'lengths': [], 'mean': -1}
        self.doc_lengths[doc_id]['lengths'].append(int(doc_length))

    def get_mean_lengths(self, doc_id =''):
        my_mean_len = 0
        others_mean_len = []
        for d_id in self.doc_lengths:
            if self.doc_lengths[d_id]['mean'] == -1:
                dlen = self.doc_lengths[d_id]['lengths']
                doc_mean_len = float(sum(dlen)) / len(dlen)
                self.doc_lengths[d_id]['mean'] = doc_mean_len
            else:
                doc_mean_len = self.doc_lengths[d_id]['mean']
            if d_id == doc_id:
                my_mean_len = doc_mean_len
            else:
                others_mean_len.append(doc_mean_len)
        oth_mean_len = float(sum(others_mean_len)) / len(others_mean_len)
        return (my_mean_len, oth_mean_len)

    def add_doc(self, doc_id ='', doc_terms=[], doc_length=-1):
        '''
        Add new document to our Language Model (training phase)
        doc_id is used here, so we build seperate LF for each doc_id
        I.e. if you call it more than once with same doc_id,
        then all terms given via doc_terms will contribute to same LM
        doc_terms: list of words in document to be added
        doc_length: the length of the document, you can provide it yourself,
                    otherwise, we use len(doc_terms) instead.
        '''
        if doc_length == -1:
            self.update_lengths(doc_id=doc_id, doc_length=len(doc_terms))
        else:
            self.update_lengths(doc_id=doc_id, doc_length=int(doc_length))
        for term in doc_terms:
            self.vocabulary.add(term)
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        self.update_counts(doc_id, ngrams)

    def laplace(self, x, y, doc_id):
        ''' Laplace (Add One/Gamma) Smoothing
        '''
        add_numer = 1 * self.laplace_gama
        laplace_mode = 'n-1'
        if laplace_mode == 'ch':
            v = len(self.vocabulary)
        elif laplace_mode == 'ch^n-1':
            v = math.pow(len(self.vocabulary),self.n-1)
        else:
            if doc_id:
                v = len(self.term_count_n_1[doc_id]['ngrams'])
            else:
                v = len(self.corpus_count_n_1['ngrams'])
        add_denom = v * self.laplace_gama
        return float(x + add_numer) / float(y + add_denom)

    def witten(self, count, n, t, log, new_doc):
        ''' Witten-Bell Smoothing
        '''
        self.return_unseen = True if new_doc else False
        if count:
            return float(count) / (n+t)
        elif self.return_unseen:
            return float(t) / (n+t)
        elif log:
            return 1
        else:
            return 1

    def pr_ngram(self, doc_id, ngram, new_doc=False, log=True, logbase=2, doc_length=-1):
        ngram_n = self.joiner.join(ngram)
        ngram_n_1 = self.joiner.join(ngram[:-1])
        ngram_n_count = self.term_count_n[doc_id]['ngrams'].get(ngram_n,0)
        ngram_n_1_count = self.term_count_n_1[doc_id]['ngrams'].get(ngram_n_1,0)
        # Apply smoothing
        if self.smoothing == 'Laplace':
            pr = self.laplace(ngram_n_count, ngram_n_1_count, doc_id)
            if self.corpus_mix == 'c' or self.corpus_mix == 'l':
                corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
                corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
                pr_dash = self.laplace(corpus_ngram_n_count - ngram_n_count,
                                  corpus_ngram_n_1_count - ngram_n_1_count,
                                  doc_id='')
                pr_mix = float(pr) / float(pr_dash)
            elif type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr
        elif self.smoothing == 'Witten':
            wittenn = ngram_n_1_count
            wittent = len([key for key in self.term_count_n[doc_id]['ngrams'] if key.startswith(ngram_n_1)])
            pr = self.witten(ngram_n_count, wittenn, wittent, log, new_doc)
            pr_mix = pr
        else:
            pr = float(ngram_n_count) / float(ngram_n_1_count)
            if type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr

        # Update seen/unseen counts
        if ngram_n_count:
            seen = True
        else:
            seen = False
        if log:
            return (-math.log(pr_mix,logbase),seen)
        else:
            return (pr_mix,seen)

    def pr_corpus(self, ngram_n, ngram_n_1):
        if self.corpus_mode == 'Hiemstra':
            df = 0
            df_total = 0
            for doc_id in self.term_count_n_1:
                if self.term_count_n[doc_id]['ngrams'].get(ngram_n,0):
                    df += 1
                df_total += 1
            pr = float(df) / float(df_total)
            if not pr:
                pr += 0.0001
        else:
            corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
            corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
            pr = self.laplace(corpus_ngram_n_count,
                              corpus_ngram_n_1_count,
                              doc_id='')
        return pr

    def pr_doc(self, doc_id, log=True, logbase=2, doc_length=-1):
        ''' This method may be overridden by implementers
            Here we assume uniform Pr(doc) within Bayes rule
            i.e. Pr(doc/string) = Pr(string/doc)
            rather than Pr(doc/string) = Pr(string/doc) * Pr(doc)
        '''
        if log:
            return 0
        else:
            return 1

    def calculate(self, doc_terms=[], actual_id='', doc_length=-1):
        '''
        Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }
        '''
        calculated = {
            'prob': -1,
            'calc_id': '',
            'actual_id': actual_id,
            'seen_unseen_count': (0,0)
        }
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        for doc_id in self.term_count_n:
            doc_pr = 0
            new_doc = True
            seen_count = 0
            unseen_count = 0
            for ngram in ngrams:
                (ngram_pr, ngram_seen) = self.pr_ngram(doc_id, ngram,
                        new_doc=new_doc, log=True, logbase=2, doc_length=doc_length)
                doc_pr += ngram_pr
                new_doc = False
                if ngram_seen:
                    seen_count += 1
                else:
                    unseen_count += 1
            doc_pr += self.pr_doc(doc_id, doc_length=doc_length)
            if self.verbose:
                print doc_id, actual_id, doc_pr
            if calculated['prob'] == -1 or doc_pr < calculated['prob']:
                calculated['prob'] = doc_pr
                calculated['calc_id'] = doc_id
                calculated['seen_unseen_count'] = (seen_count, unseen_count)
        self.unseen_counts.per_doc(doc_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        self.unseen_counts.per_cic(calculated_id=calculated['calc_id'],
            actual_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        return calculated

if __name__ == '__main__':

    #Preprocessor
    def term2ch(text):
        return [ch for ch in text]

    lm = LM(n=2, verbose=True, smoothing='Laplace', corpus_mix=0)

    #print lm.to_ngrams(term2ch('hello dear world'))

    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i see glass of apple juice'))
    lm.add_doc(doc_id='apple', doc_terms=term2ch('the tree is full or apples'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i see the orange tree shaking'))
    lm.add_doc(doc_id='orange', doc_terms=term2ch('orange orange juice'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i do not like jaffa cake'))
    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i have apple juice'))
    #print lm.pr
    result = lm.calculate(doc_terms=term2ch('orango juice'))
    print result

    print lm.term_count_n
    print lm.term_count_n_1
    print lm.vocabulary
    print lm.overlaps()


        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams

'''
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model
'''

# Author: Tarek Amr <@gr33ndata>

import sys, math

class UnseenTerms:

    def __init__(self):
        # Count unseen n-grams per each class
        self.doc_counts = {}
        # Count unseen n-grams per correct/incorrect results
        self.cic_counts = {}

    def _seen_unseel_label(self, seen, unseen):
        if seen + unseen:
            label = '%0.2f' % (float(unseen) / (seen+unseen))
        else:
            label = '%0.2f' %  0
        return label

    def per_cic(self, calculated_id='', actual_id='', seen_unseen=None):
        calculated_id = str(calculated_id).strip()
        actual_id =str(actual_id).strip()
        unseen_ratio_str = self._seen_unseel_label(*seen_unseen)
        if not unseen_ratio_str in self.cic_counts:
            self.cic_counts[unseen_ratio_str] = [0,0]
        if calculated_id == actual_id:
            self.cic_counts[unseen_ratio_str][0] += 1
        else:
            self.cic_counts[unseen_ratio_str][1] += 1

    def per_doc(self, doc_id='', seen_unseen=None):
        if not doc_id in self.doc_counts:
            self.doc_counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }
        self.doc_counts[doc_id]['seen'] += seen_unseen[0]
        self.doc_counts[doc_id]['unseen'] += seen_unseen[1]

    def display(self, per_doc=True, per_cic=True):
        if per_doc:
            print '\nUnseen n-grams as per actual doc_ids'
            doc_ids = self.doc_counts.keys()
            doc_ids.sort()
            for doc_id in doc_ids:
                unseen = self.doc_counts[doc_id]['unseen']
                total = self.doc_counts[doc_id]['unseen'] + self.doc_counts[doc_id]['seen']
                unseen_ratio =  float(unseen) / total
                print 'Ratio of unseen (%s): %0.4f' % (doc_id, unseen_ratio)
        if per_cic:
            print ''
            print 'Correct/Incorrect Unseen Ratios'
            cic_x = []
            correct = []
            incorrect = []
            cic_labels = self.cic_counts.keys()
            cic_labels.sort()
            for label in cic_labels:
                cic_x.append(str(label))
                correct.append(str(self.cic_counts[label][0]))
                incorrect.append(str(self.cic_counts[label][1]))
            print 'CIC X = [' + ','.join(cic_x) + ']'
            print 'Correct   = [' + ','.join(correct) + ']'
            print 'Incorrect = [' + ','.join(incorrect) + ']'

class LM:

    def __init__(self, n=3, lpad='', rpad='',
                 smoothing='Laplace', laplace_gama=1,
                 corpus_mix=0, corpus_mode='Miller',
                 verbose=False):
        '''
        Initialize our LM
        n: Order of ngram LM, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding.
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        smoothing: 'Laplace' or 'Witten'
        laplace_gama: Multiply 1 and V by this factor gamma
        corpus_mix: 0 (default) only use document probabilites
                  : or value between 0 and 1 lambda
                  : or c (or l) for Log-likelihood Odds model (Cooper LDA)
        corpus_mode: 'Hiemstra' or 'Miller'
                   : This tell us how to calculate pr_corpus(t)
                   : In fact, 'Hiemstra' makes sense only with Multivaria LM
                   : Whereas the LM here is Multinomial.
        '''
        if n < 1:
            raise ValueError('n should be >= 1')
        self.n = n
        # Counters for joint probabilities
        self.term_count_n = {}
        self.term_count_n_1 = {}
        # To be used in case of mixing doc prob with corpus prob.
        self.corpus_count_n = {'ngrams': {}, 'total': 0}
        self.corpus_count_n_1 = {'ngrams': {}, 'total': 0}
        self.doc_lengths = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.smoothing = smoothing
        self.laplace_gama = float(laplace_gama)
        if not type(corpus_mix) is str:
            self.corpus_mix = min(float(corpus_mix),1)
        else:
            self.corpus_mix = corpus_mix
        self.corpus_mode = corpus_mode
        self.joiner = ''
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def get_vocabulary(self):
        return self.vocabulary

    def display(self, per_doc=True, per_cic=True):
        '''
        Displays statistics about our LM
        '''
        voc_list = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            ngrams = len(self.term_count_n[doc_id]['ngrams'])
            print 'n-Grams (doc %s): %d' % (str(doc_id), ngrams)
            ngrams1 = len(self.term_count_n_1[doc_id]['ngrams'])
            print '(n-1)-Grams (doc %s): %d' % (str(doc_id), ngrams1)
            voc_list.append(ngrams)
        print 'Classed Vocabularies:', voc_list
        print ''
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        print 'n-Grams (collection): %d' % (corpus_ngrams)
        corpus_ngrams1 = len(self.corpus_count_n_1['ngrams'])
        print '(n-1)-Grams (collection): %d' % (corpus_ngrams1)
        self.unseen_counts.display(per_doc, per_cic)
        # Display overlapping n-grams between classes

    def get_ngram_counts(self):
        ''' Returns a list of n-gram counts
            Array of classes counts and last item is for corpus
        '''
        ngram_counts = {
            'classes': [],
            'classes-1': [],
            'corpus': 0,
            'corpus-1': 0
        }
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            #print self.term_count_n[doc_id]
            class_ngrams = len(self.term_count_n[doc_id]['ngrams'])
            class_n1grams = len(self.term_count_n_1[doc_id]['ngrams'])
            ngram_counts['classes'].append(class_ngrams)
            ngram_counts['classes-1'].append(class_n1grams)
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        corpus_n1grams = len(self.corpus_count_n_1['ngrams'])
        ngram_counts['corpus'] = corpus_ngrams
        ngram_counts['corpus-1'] = corpus_n1grams
        return ngram_counts

    def overlaps(self):
        ''' Print overlapping n-grams between classes.
        '''
        omx = []
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            row = [0] * len(doc_ids)
            omx.append(row)

        for i, doc_id_i in enumerate(doc_ids):

            ngrams = len(self.term_count_n[doc_id_i]['ngrams'])
            omx[i][i] = ngrams
            for j in range(i):
                doc_id_j = doc_ids[j]
                ongrams = 0
                for ngram in self.term_count_n[doc_id_i]['ngrams']:
                    if ngram in self.term_count_n[doc_id_j]['ngrams']:
                        ongrams += 1
                    omx[i][j] = 0 #ongrams
                    omx[j][i] = ongrams
        print '\nn-gram overlaps:'
        print doc_ids
        for i in range(len(omx)):
            row = []
            for j in range(len(omx[i])):
                row.append( round( float(omx[i][j] * 100) / omx[i][i],2 ) )
            print row

    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams
            terms: list of terms
        '''

    def lr_padding(self, terms):
        '''
        Pad doc from the left and right before adding,
        depending on what's in self.lpad and self.rpad
        If any of them is '', then don't pad there.
        '''
        lpad = rpad = []
        if self.lpad:
            lpad = [self.lpad] * (self.n - 1)
        if self.rpad:
            rpad = [self.rpad] * (self.n - 1)
        return lpad + terms + rpad

    def update_counts(self, doc_id ='', ngrams=[]):
        if not doc_id in self.term_count_n:
            self.term_count_n[doc_id] = {'ngrams': {}, 'total': 0}
        if not doc_id in self.term_count_n_1:
            self.term_count_n_1[doc_id] = {'ngrams': {}, 'total': 0}
        for ngram in ngrams:
            # Generate n-grams and sub-ngrams
            ngram_n = self.joiner.join(ngram)
            ngram_n_1 = self.joiner.join(ngram[:-1])
            # Update n-gram counts for doc_id
            if ngram_n in self.term_count_n[doc_id]['ngrams']:
                self.term_count_n[doc_id]['ngrams'][ngram_n] += 1
            else:
                self.term_count_n[doc_id]['ngrams'][ngram_n] = 1
            self.term_count_n[doc_id]['total'] += 1
            # Update n-gram counts for corpus
            if ngram_n in self.corpus_count_n['ngrams']:
                self.corpus_count_n['ngrams'][ngram_n] += 1
            else:
                self.corpus_count_n['ngrams'][ngram_n] = 1
            self.corpus_count_n['total'] += 1
            if ngram_n_1 in self.term_count_n_1[doc_id]['ngrams']:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] += 1
            else:
                self.term_count_n_1[doc_id]['ngrams'][ngram_n_1] = 1
            self.term_count_n_1[doc_id]['total'] += 1
            if ngram_n_1 in self.corpus_count_n_1['ngrams']:
                self.corpus_count_n_1['ngrams'][ngram_n_1] += 1
            else:
                self.corpus_count_n_1['ngrams'][ngram_n_1] = 1
            self.corpus_count_n_1['total'] += 1

    def update_lengths(self, doc_id ='', doc_length=0):
        if not doc_id in self.doc_lengths:
            self.doc_lengths[doc_id] = {'lengths': [], 'mean': -1}
        self.doc_lengths[doc_id]['lengths'].append(int(doc_length))

    def get_mean_lengths(self, doc_id =''):
        my_mean_len = 0
        others_mean_len = []
        for d_id in self.doc_lengths:
            if self.doc_lengths[d_id]['mean'] == -1:
                dlen = self.doc_lengths[d_id]['lengths']
                doc_mean_len = float(sum(dlen)) / len(dlen)
                self.doc_lengths[d_id]['mean'] = doc_mean_len
            else:
                doc_mean_len = self.doc_lengths[d_id]['mean']
            if d_id == doc_id:
                my_mean_len = doc_mean_len
            else:
                others_mean_len.append(doc_mean_len)
        oth_mean_len = float(sum(others_mean_len)) / len(others_mean_len)
        return (my_mean_len, oth_mean_len)

    def add_doc(self, doc_id ='', doc_terms=[], doc_length=-1):
        '''
        Add new document to our Language Model (training phase)
        doc_id is used here, so we build seperate LF for each doc_id
        I.e. if you call it more than once with same doc_id,
        then all terms given via doc_terms will contribute to same LM
        doc_terms: list of words in document to be added
        doc_length: the length of the document, you can provide it yourself,
                    otherwise, we use len(doc_terms) instead.
        '''
        if doc_length == -1:
            self.update_lengths(doc_id=doc_id, doc_length=len(doc_terms))
        else:
            self.update_lengths(doc_id=doc_id, doc_length=int(doc_length))
        for term in doc_terms:
            self.vocabulary.add(term)
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        self.update_counts(doc_id, ngrams)

    def laplace(self, x, y, doc_id):
        ''' Laplace (Add One/Gamma) Smoothing
        '''
        add_numer = 1 * self.laplace_gama
        laplace_mode = 'n-1'
        if laplace_mode == 'ch':
            v = len(self.vocabulary)
        elif laplace_mode == 'ch^n-1':
            v = math.pow(len(self.vocabulary),self.n-1)
        else:
            if doc_id:
                v = len(self.term_count_n_1[doc_id]['ngrams'])
            else:
                v = len(self.corpus_count_n_1['ngrams'])
        add_denom = v * self.laplace_gama
        return float(x + add_numer) / float(y + add_denom)

    def witten(self, count, n, t, log, new_doc):
        ''' Witten-Bell Smoothing
        '''
        self.return_unseen = True if new_doc else False
        if count:
            return float(count) / (n+t)
        elif self.return_unseen:
            return float(t) / (n+t)
        elif log:
            return 1
        else:
            return 1

    def pr_ngram(self, doc_id, ngram, new_doc=False, log=True, logbase=2, doc_length=-1):
        ngram_n = self.joiner.join(ngram)
        ngram_n_1 = self.joiner.join(ngram[:-1])
        ngram_n_count = self.term_count_n[doc_id]['ngrams'].get(ngram_n,0)
        ngram_n_1_count = self.term_count_n_1[doc_id]['ngrams'].get(ngram_n_1,0)
        # Apply smoothing
        if self.smoothing == 'Laplace':
            pr = self.laplace(ngram_n_count, ngram_n_1_count, doc_id)
            if self.corpus_mix == 'c' or self.corpus_mix == 'l':
                corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
                corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
                pr_dash = self.laplace(corpus_ngram_n_count - ngram_n_count,
                                  corpus_ngram_n_1_count - ngram_n_1_count,
                                  doc_id='')
                pr_mix = float(pr) / float(pr_dash)
            elif type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr
        elif self.smoothing == 'Witten':
            wittenn = ngram_n_1_count
            wittent = len([key for key in self.term_count_n[doc_id]['ngrams'] if key.startswith(ngram_n_1)])
            pr = self.witten(ngram_n_count, wittenn, wittent, log, new_doc)
            pr_mix = pr
        else:
            pr = float(ngram_n_count) / float(ngram_n_1_count)
            if type(float(self.corpus_mix)) is float and self.corpus_mix > 0:
                pr_corpus = self.pr_corpus(ngram_n, ngram_n_1)
                pr_mix = self.corpus_mix * pr_corpus + (1 - self.corpus_mix) * pr
            else:
                pr_mix = pr

        # Update seen/unseen counts
        if ngram_n_count:
            seen = True
        else:
            seen = False
        if log:
            return (-math.log(pr_mix,logbase),seen)
        else:
            return (pr_mix,seen)

    def pr_corpus(self, ngram_n, ngram_n_1):
        if self.corpus_mode == 'Hiemstra':
            df = 0
            df_total = 0
            for doc_id in self.term_count_n_1:
                if self.term_count_n[doc_id]['ngrams'].get(ngram_n,0):
                    df += 1
                df_total += 1
            pr = float(df) / float(df_total)
            if not pr:
                pr += 0.0001
        else:
            corpus_ngram_n_count = self.corpus_count_n['ngrams'].get(ngram_n,0)
            corpus_ngram_n_1_count = self.corpus_count_n_1['ngrams'].get(ngram_n_1,0)
            pr = self.laplace(corpus_ngram_n_count,
                              corpus_ngram_n_1_count,
                              doc_id='')
        return pr

    def pr_doc(self, doc_id, log=True, logbase=2, doc_length=-1):
        ''' This method may be overridden by implementers
            Here we assume uniform Pr(doc) within Bayes rule
            i.e. Pr(doc/string) = Pr(string/doc)
            rather than Pr(doc/string) = Pr(string/doc) * Pr(doc)
        '''
        if log:
            return 0
        else:
            return 1

    def calculate(self, doc_terms=[], actual_id='', doc_length=-1):
        '''
        Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }
        '''
        calculated = {
            'prob': -1,
            'calc_id': '',
            'actual_id': actual_id,
            'seen_unseen_count': (0,0)
        }
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms)
        for doc_id in self.term_count_n:
            doc_pr = 0
            new_doc = True
            seen_count = 0
            unseen_count = 0
            for ngram in ngrams:
                (ngram_pr, ngram_seen) = self.pr_ngram(doc_id, ngram,
                        new_doc=new_doc, log=True, logbase=2, doc_length=doc_length)
                doc_pr += ngram_pr
                new_doc = False
                if ngram_seen:
                    seen_count += 1
                else:
                    unseen_count += 1
            doc_pr += self.pr_doc(doc_id, doc_length=doc_length)
            if self.verbose:
                print doc_id, actual_id, doc_pr
            if calculated['prob'] == -1 or doc_pr < calculated['prob']:
                calculated['prob'] = doc_pr
                calculated['calc_id'] = doc_id
                calculated['seen_unseen_count'] = (seen_count, unseen_count)
        self.unseen_counts.per_doc(doc_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        self.unseen_counts.per_cic(calculated_id=calculated['calc_id'],
            actual_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        return calculated

if __name__ == '__main__':

    #Preprocessor
    def term2ch(text):
        return [ch for ch in text]

    lm = LM(n=2, verbose=True, smoothing='Laplace', corpus_mix=0)

    #print lm.to_ngrams(term2ch('hello dear world'))

    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i see glass of apple juice'))
    lm.add_doc(doc_id='apple', doc_terms=term2ch('the tree is full or apples'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i see the orange tree shaking'))
    lm.add_doc(doc_id='orange', doc_terms=term2ch('orange orange juice'))
    #lm.add_doc(doc_id='orange', doc_terms=term2ch('i do not like jaffa cake'))
    #lm.add_doc(doc_id='apple', doc_terms=term2ch('i have apple juice'))
    #print lm.pr
    result = lm.calculate(doc_terms=term2ch('orango juice'))
    print result

    print lm.term_count_n
    print lm.term_count_n_1
    print lm.vocabulary
    print lm.overlaps()


# line_plot_demo.py

from reportlab.lib import colors
from reportlab.graphics.shapes import Drawing
from reportlab.graphics.charts.lineplots import LinePlot
from reportlab.graphics.widgets.markers import makeMarker

d = Drawing(400, 400)
    line = LinePlot()
    line.x = 50
    line.y = 85
    line.height = 150
    line.width = 250
    line.lineLabelFormat = '%2.0f'

    data = [
            ((1,1), (2,2), (2.5,1), (3,3), (4,5)),
            ((1,2), (2,3), (2.5,2), (3.5,5), (4,6))
        ]
    line.data = data

    line.lines[0].strokeColor = colors.green
    line.lines[1].strokeColor = colors.blue
    line.lines[0].strokeWidth = 3

    line.lines[0].symbol = makeMarker('Circle')
    line.lines[1].symbol = makeMarker('Hexagon')

    line.xValueAxis.valueMin = 0
    line.xValueAxis.valueMax = 10
    line.xValueAxis.valueSteps = [1, 2, 4]
    line.xValueAxis.labelTextFormat = '%2.1f'

    line.yValueAxis.valueMin = 0
    line.yValueAxis.valueMax = 12

    d.add(line, '')
    d.save(formats=['pdf'], outDir='.', fnRoot='line_plot_demo')

if __name__ == '__main__':
    line_plot_demo()


# line_plot_demo.py

from reportlab.lib import colors
from reportlab.graphics.shapes import Drawing
from reportlab.graphics.charts.lineplots import LinePlot
from reportlab.graphics.widgets.markers import makeMarker

def line_plot_demo():
    d = Drawing(400, 400)
    line = LinePlot()
    line.x = 50
    line.y = 85
    line.height = 150
    line.width = 250
    line.lineLabelFormat = '%2.0f'

    data = [
            ((1,1), (2,2), (2.5,1), (3,3), (4,5)),
            ((1,2), (2,3), (2.5,2), (3.5,5), (4,6))
        ]
    line.data = data

    line.lines[0].strokeColor = colors.green
    line.lines[1].strokeColor = colors.blue
    line.lines[0].strokeWidth = 3

    line.lines[0].symbol = makeMarker('Circle')
    line.lines[1].symbol = makeMarker('Hexagon')

    line.xValueAxis.valueMin

    line.xValueAxis.valueMax = 10
    line.xValueAxis.valueSteps = [1, 2, 4]
    line.xValueAxis.labelTextFormat = '%2.1f'

    line.yValueAxis.valueMin = 0
    line.yValueAxis.valueMax = 12

    d.add(line, '')
    d.save(formats=['pdf'], outDir='.', fnRoot='line_plot_demo')

if __name__ == '__main__':
    line_plot_demo()


from uniborg.util import lightning_cmd

@borg.on(lightning_cmd(pattern="f"))

paytext = event.pattern_match.group(1)
    pay = "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}".format(
        paytext * 8,
        paytext * 8,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 6,
        paytext * 6,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
    )
    await event.edit(pay)


paytext = event.pattern_match.group(1)
    pay = "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}".format(
        paytext * 8,
        paytext * 8,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 6,
        paytext * 6,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
    )
    await event.edit(pay)

from uniborg.util import lightning_cmd

@borg.on(lightning_cmd(pattern="f"))
async def payf(event):


from uniborg.util import lightning_cmd

@borg.on(lightning_cmd(pattern="f"))
async def payf(event):
    paytext = event.pattern_match.group(1)
    pay = "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}".format(
        paytext * 8,
        paytext * 8,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 6,
        paytext * 6,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
        paytext * 2,
    )
    await event.edit


from math import exp
listf100=[]
def logreg(data=listf100, g=listf100, w=listf100, dims=int):
    dot = 0.0
    j = 0
    while(j<dims):
        dot += data[j]*w[j]
        j+=1
    label = data[dims]
    scale = (1.0 / (1.0+ exp(-label*dot)) - 1.0) * label
    j2 = 0
    while(j2<dims):
        g[j2]+=scale*data[j2]
        j2+=1

data = range(100)
    g = range(100)
    w = range(100)
    dims = 99
    logreg(data, g, w, dims)


data = range(100)
    g = range(100)
    w = range(100)
    dims = 99
    logreg(data, g, w, dims)

from math import exp
listf100=[]
def logreg(data=listf100, g=listf100, w=listf100, dims=int):
    dot = 0.0
    j = 0
    while(j<dims):
        dot += data[j]*w[j]
        j+=1
    label = data[dims]
    scale = (1.0 / (1.0+ exp(-label*dot)) - 1.0) * label
    j2 = 0
    while(j2<dims):
        g[j2]+=scale*data[j2]
        j2+=1

def run():


from math import exp
listf100=[]
def logreg(data=listf100, g=listf100, w=listf100, dims=int):
    dot = 0.0
    j = 0
    while(j<dims):
        dot += data[j]*w[j]
        j+=1
    label = data[dims]
    scale = (1.0 / (1.0+ exp(-label*dot)) - 1.0) * label

    while(j2<dims):
        g[j2]+=scale*data[j2]
        j2+=1

def run():
    data = range(100)
    g = range(100)
    w = range(100)
    dims = 99
    logreg(data, g, w, dims)


while(j2<dims):
        g[j2]+=scale*data[j2]
        j2+=1

from math import exp
listf100=[]
def logreg(data=listf100, g=listf100, w=listf100, dims=int):
    dot = 0.0
    j = 0
    while(j<dims):
        dot += data[j]*w[j]
        j+=1
    label = data[dims]
    scale = (1.0 / (1.0+ exp(-label*dot)) - 1.0) * label
    j2 = 0

def run():
    data = range(100)
    g = range(100)
    w = range(100)
    dims = 99
    logreg(data, g, w, dims)