if value:
                return re.sub('[\n\t\r]', '', value)
            return ""

# This file is part of Viper - http://example.com

import os
import re
import email
import hashlib
import tempfile
import olefile

from lib.common.abstracts import Module
from lib.core.session import __sessions__

class EmailParse(Module):
    cmd = 'email'
    description = 'Parse eml and msg email files'
    authors = ['Kevin Breen', 'nex']

    def __init__(self):
        super(EmailParse, self).__init__()
        self.parser.add_argument('-e', '--envelope', action='store_true', help='Show the email envelope')
        self.parser.add_argument('-f', '--attach', action='store_true', help='Show Attachment information')
        self.parser.add_argument('-r', '--header', action='store_true', help='Show email Header information')
        self.parser.add_argument('-t', '--trace', action='store_true', help='Show email path via Received headers')
        self.parser.add_argument('-T', '--traceall', action='store_true', help='Show email path via verbose Received headers')
        self.parser.add_argument('-s', '--spoofcheck', action='store_true', help='Test email for possible spoofing')
        self.parser.add_argument('-a', '--all', action='store_true', help='Run all the options')
        self.parser.add_argument('-o', '--open', type=int, help='Switch session to the specified attachment')

    def run(self, *args):

        def string_clean(value):

        def parse_ole_msg(ole):
            stream_dirs = ole.listdir()
            for stream in stream_dirs:
                # get stream that contains the email header
                if stream[0].startswith('__substg1.0_007D'):
                    email_header = ole.openstream(stream).read()
                    if stream[0].endswith('001F'):  # Unicode probably needs something better than just stripping \x00
                        email_header = email_header.replace('\x00', '')
            # If it came from outlook we may need to trim some lines
            try:
                email_header = email_header.split('Version 2.0\x0d\x0a', 1)[1]
            except:
                pass

            # Leaving us an RFC compliant email to parse
            msg = email.message_from_string(email_header)
            return msg

        def parse_ole_attachments(ole):
            # Hard part now, each part of the attachment is in a seperate stream

            # need to get a unique stream id for each att
            # its in the streamname as an 8 digit number.
            for i in range(20):  # arbitrary count of emails. i dont expecet this many
                stream_number = str(i).zfill(8)
                stream_name = '__attach_version1.0_#' + stream_number
                # Unicode
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass
                # ASCII
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass

        def att_session(att_id, msg, ole_flag):
            att_count = 0
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_filename = att_filename.replace('\x00', '')
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    if i == att_id:
                        self.log('info', "Switching session to {0}".format(att_filename))
                        tmp_path = os.path.join(tempfile.gettempdir(), att_filename)
                        with open(tmp_path, 'w') as tmp:
                            tmp.write(att_data)
                        __sessions__.new(tmp_path)
                        return

            else:
                for part in msg.walk():
                    if part.get_content_type() == 'message/rfc822':
                        rfc822 = True
                    else:
                        rfc822 = False

                    if part.get_content_maintype() == 'multipart' \
                        or not part.get('Content-Disposition') \
                            and not rfc822:
                        continue

                    att_count += 1
                    if att_count == att_id:
                        if rfc822:
                            data = part.as_string()
                            m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", data, flags=re.S)
                            if not m:
                                self.log('error', "Could not extract RFC822 formatted message")
                                return
                            data = m.group(1)
                            att_size = len(data)
                            filename = "rfc822msg_{0}.eml".format(att_size)
                        else:
                            data = part.get_payload(decode=True)
                            filename = part.get_filename()

                        self.log('info', "Switching session to {0}".format(filename))

                        if data:
                            tmp_path = os.path.join(tempfile.gettempdir(), filename)
                            with open(tmp_path, 'w') as tmp:
                                tmp.write(data)
                            __sessions__.new(tmp_path)
                            return

        def email_envelope(msg):
            # Envelope
            self.log('info', "Email envelope:")
            rows = [
                ['Subject', msg.get("Subject")],
                ['To', msg.get("To")],
                ['From', msg.get("From")],
                ['Cc', msg.get("Cc")],
                ['Bcc', msg.get("Bcc")],
                ['Date', msg.get("Date")]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_header(msg):
            # Headers
            rows = []
            for x in msg.keys():
                # Adding Received to ignore list. this has to be handeled separately if there are more then one line
                if x not in ['Subject', 'From', 'To', 'Date', 'Cc', 'Bcc', 'DKIM-Signature', 'Received']:
                    rows.append([x, string_clean(msg.get(x))])
            for x in msg.get_all('Received'):
                rows.append(['Received', string_clean(x)])
            self.log('info', "Email headers:")
            rows = sorted(rows, key=lambda entry: entry[0])
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_trace(msg, verbose):
            rows = []
            if verbose:
                fields = ['from', 'by', 'with', 'id', 'for', 'timestamp']
            else:
                fields = ['from', 'by', 'timestamp']
            for x in msg.get_all('Received'):
                x = string_clean(x)
                cre = re.compile("""
                    (?: from \s+ (?P<from>.*?) (?=by|with|id|ID|for|;|$) )?
                    (?: by \s+ (?P<by>.*?) (?=with|id|ID|for|;|$) )?
                    (?: with \s+ (?P<with>.*?) (?=id|ID|for|;|$) )?
                    (?: (id|ID) \s+ (?P<id>.*?) (?=for|;|$) )?
                    (?: for \s+ (?P<for>.*?) (?=;|$) )?
                    (?: \s* ; \s* (?P<timestamp>.*) )?
                    """, flags=re.X | re.I)
                m = cre.search(x)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                t = []
                for groupname in fields:
                    t.append(string_clean(m.group(groupname)))
                rows.insert(0, t)
            self.log('info', "Email path trace:")
            self.log('table', dict(header=fields, rows=rows))
            return

        def email_spoofcheck(msg, dnsenabled):
            self.log('info', "Email spoof check:")

            # test 1: check if From address is the same as Sender, Reply-To, and Return-Path
            rows = [
                ['Sender', string_clean(msg.get("Sender"))],
                ['From', string_clean(msg.get("From"))],
                ['Reply-To', string_clean(msg.get("Reply-To"))],
                ['Return-Path', string_clean(msg.get("Return-Path"))]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            addr = {
                'Sender': email.utils.parseaddr(string_clean(msg.get("Sender")))[1],
                'From': email.utils.parseaddr(string_clean(msg.get("From")))[1],
                'Reply-To': email.utils.parseaddr(string_clean(msg.get("Reply-To")))[1],
                'Return-Path': email.utils.parseaddr(string_clean(msg.get("Return-Path")))[1]
            }
            if (addr['From'] == ''):
                self.log('error', "No From address!")
                return
            elif addr['Sender'] and (addr['From'] != addr['Sender']):
                self.log('warning', "Email FAILED: From address different than Sender")
            elif addr['Reply-To'] and (addr['From'] != addr['Reply-To']):
                self.log('warning', "Email FAILED: From address different than Reply-To")
            elif addr['Return-Path'] and (addr['From'] != addr['Return-Path']):
                self.log('warning', "Email FAILED: From address different than Return-Path")
            else:
                self.log('success', "Email PASSED: From address the same as Sender, Reply-To, and Return-Path")

            # test 2: check to see if first Received: by domain matches sender MX domain
            if not dnsenabled:
                self.log('info', "Unable to run Received by / sender check without dnspython available")
            else:
                r = msg.get_all('Received')[-1]
                m = re.search("by\s+(\S*?)(?:\s+\(.*?\))?\s+with", r)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                byname = m.group(1)
                # this can be either a name or an IP
                m = re.search("(\w+\.\w+|\d+\.\d+\.\d+\.\d+)$", byname)
                if not m:
                    self.log('error', "Could not find domain or IP in Received by field")
                    return
                bydomain = m.group(1)
                domains = [['Received by', bydomain]]
                # if it's an IP, do the reverse lookup
                m = re.search("\.\d+$", bydomain)
                if m:
                    bydomain = str(dns.reversename.from_address(bydomain)).strip('.')
                    domains.append(['Received by reverse lookup', bydomain])
                # if the email has a Sender header, use that
                if (addr['Sender'] != ""):
                    m = re.search("(\w+\.\w+)$", addr['Sender'])
                    if not m:
                        self.log('error', "Sender header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['Sender', fromdomain])
                # otherwise, use the From header
                else:
                    m = re.search("(\w+\.\w+)$", addr['From'])
                    if not m:
                        self.log('error', "From header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['From', fromdomain])

                bymatch = False
                try:
                    mx = dns.resolver.query(fromdomain, 'MX')
                    if mx :
                        for rdata in mx:
                            m = re.search("(\w+\.\w+).$", str(rdata.exchange))
                            if not m:
                                self.log('error', "MX domain regex didn't match")
                                continue
                            domains.append(['MX for ' + fromdomain, m.group(1)])
                            if bydomain == m.group(1):
                                bymatch = True
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                except:
                    domains.append(['MX for ' + fromdomain, "not registered in DNS"])
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                if bymatch:
                    self.log('success', "Email PASSED: Received by domain found in Sender/From MX domains")
                else:
                    self.log('warning', "Email FAILED: Could not match Received by domain to Sender/From MX")

            # test 3: look at SPF records
            rspf = []
            results = set()
            allspf = msg.get_all('Received-SPF')
            if not allspf:
                return
            for spf in allspf:
                m = re.search("\s*(\w+)\s+\((.*?):\s*(.*?)\)\s+(.*);", string_clean(spf))
                if not m:
                    self.log('error', "Received-SPF regex didn't match")
                    return
                rspf.append([m.group(2), m.group(1), m.group(3), m.group(4)])
                results = results | {m.group(1)}
            self.log('table', dict(header=['Domain', 'Action', 'Info', 'Additional'], rows=rspf))
            if results & {'fail', 'softfail'}:
                self.log('warning', "Email FAILED: Found fail or softfail SPF results")
            elif results & {'none', 'neutral'}:
                self.log('warning', "Email NEUTRAL: Found none or neutral SPF results")
            elif results & {'permerror', 'temperror'}:
                self.log('warning', "Email NEUTRAL: Found error condition")
            elif results & {'pass'}:
                self.log('success', "Email PASSED: Found SPF pass result")

            return

        def email_attachments(msg, ole_flag):
            # Attachments
            att_count = 0
            rows = []
            links = []
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass

            else:
                # Walk through email string.
                for part in msg.walk():
                    content_type = part.get_content_type()

                    if content_type == 'multipart':
                        continue

                    if content_type in ('text/plain', 'text/html'):
                        part_content = part.get_payload(decode=True)
                        for link in re.findall(r'(https?://[^"<>\s]+)', part_content):
                            if link not in links:
                                links.append(link)

                    if content_type == 'message/rfc822':
                        part_content = part.as_string()
                        m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", part_content, flags=re.S)
                        if not m:
                            self.log('error', "Could not extract RFC822 formatted message")
                            return
                        part_content = m.group(1)
                        att_size = len(part_content)
                        att_file_name = "rfc822msg_{0}.eml".format(att_size)
                        att_md5 = hashlib.md5(part_content).hexdigest()
                        att_count += 1
                        rows.append([att_count, att_file_name, content_type, att_size, att_md5])
                        continue

                    if not part.get('Content-Disposition'):
                        # These are not attachments.
                        continue

                    att_file_name = part.get_filename()
                    att_size = len(part_content)

                    if not att_file_name:
                        continue

                    att_data = part.get_payload(decode=True)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    att_count += 1
                    rows.append([att_count, att_file_name, part.get_content_type(), att_size, att_md5])

            self.log('info', "Email attachments (total: {0}):".format(att_count))
            if att_count > 0:
                self.log('table', dict(header=['ID', 'FileName', 'Content Type', 'File Size', 'MD5'], rows=rows))

            self.log('info', "Email links:")
            for link in links:
                self.log('item', link)
            return

        # Start Here
        if not __sessions__.is_set():
            self.log('error', "No session opened")
            return

        super(EmailParse, self).run(*args)
        if self.args is None:
            return

        # see if we can load the dns library for MX lookup spoof detecton
        try:
            import dns.resolver
            import dns.reversename
            dnsenabled = True
        except ImportError:
            dnsenabled = False

        # Try to open as an ole msg, if not treat as email string
        try:
            ole = olefile.OleFileIO(__sessions__.current.file.path)
            ole_flag = True
        except:
            ole_flag = False
            email_handle = open(__sessions__.current.file.path)
            msg = email.message_from_file(email_handle)
            email_handle.close()

        if self.args.open is not None:
            if ole_flag:
                msg = ole
            att_session(self.args.open, msg, ole_flag)
        elif self.args.envelope:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
        elif self.args.attach:
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        elif self.args.header:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_header(msg)
        elif self.args.trace:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, False)
        elif self.args.traceall:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, True)
        elif self.args.spoofcheck:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_spoofcheck(msg, dnsenabled)
        elif self.args.all:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
            email_header(msg)
            email_trace(msg, True)
            email_spoofcheck(msg, dnsenabled)
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        else:
            self.log('error', 'At least one of the parameters is required')
            self.usage()


er.add_argument('-a', '--all', action='store_true', help='Run all the options')

# This file is part of Viper - http://example.com

import os
import re
import email
import hashlib
import tempfile
import olefile

from lib.common.abstracts import Module
from lib.core.session import __sessions__

class EmailParse(Module):
    cmd = 'email'
    description = 'Parse eml and msg email files'
    authors = ['Kevin Breen', 'nex']

    def __init__(self):
        super(EmailParse, self).__init__()
        self.parser.add_argument('-e', '--envelope', action='store_true', help='Show the email envelope')
        self.parser.add_argument('-f', '--attach', action='store_true', help='Show Attachment information')
        self.parser.add_argument('-r', '--header', action='store_true', help='Show email Header information')
        self.parser.add_argument('-t', '--trace', action='store_true', help='Show email path via Received headers')
        self.parser.add_argument('-T', '--traceall', action='store_true', help='Show email path via verbose Received headers')
        self.parser.add_argument('-s', '--spoofcheck', action='store_true', help='Test email for possible spoofing')
        self.pars

        self.parser.add_argument('-o', '--open', type=int, help='Switch session to the specified attachment')

    def run(self, *args):

        def string_clean(value):
            if value:
                return re.sub('[\n\t\r]', '', value)
            return ""

        def parse_ole_msg(ole):
            stream_dirs = ole.listdir()
            for stream in stream_dirs:
                # get stream that contains the email header
                if stream[0].startswith('__substg1.0_007D'):
                    email_header = ole.openstream(stream).read()
                    if stream[0].endswith('001F'):  # Unicode probably needs something better than just stripping \x00
                        email_header = email_header.replace('\x00', '')
            # If it came from outlook we may need to trim some lines
            try:
                email_header = email_header.split('Version 2.0\x0d\x0a', 1)[1]
            except:
                pass

            # Leaving us an RFC compliant email to parse
            msg = email.message_from_string(email_header)
            return msg

        def parse_ole_attachments(ole):
            # Hard part now, each part of the attachment is in a seperate stream

            # need to get a unique stream id for each att
            # its in the streamname as an 8 digit number.
            for i in range(20):  # arbitrary count of emails. i dont expecet this many
                stream_number = str(i).zfill(8)
                stream_name = '__attach_version1.0_#' + stream_number
                # Unicode
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass
                # ASCII
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass

        def att_session(att_id, msg, ole_flag):
            att_count = 0
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_filename = att_filename.replace('\x00', '')
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    if i == att_id:
                        self.log('info', "Switching session to {0}".format(att_filename))
                        tmp_path = os.path.join(tempfile.gettempdir(), att_filename)
                        with open(tmp_path, 'w') as tmp:
                            tmp.write(att_data)
                        __sessions__.new(tmp_path)
                        return

            else:
                for part in msg.walk():
                    if part.get_content_type() == 'message/rfc822':
                        rfc822 = True
                    else:
                        rfc822 = False

                    if part.get_content_maintype() == 'multipart' \
                        or not part.get('Content-Disposition') \
                            and not rfc822:
                        continue

                    att_count += 1
                    if att_count == att_id:
                        if rfc822:
                            data = part.as_string()
                            m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", data, flags=re.S)
                            if not m:
                                self.log('error', "Could not extract RFC822 formatted message")
                                return
                            data = m.group(1)
                            att_size = len(data)
                            filename = "rfc822msg_{0}.eml".format(att_size)
                        else:
                            data = part.get_payload(decode=True)
                            filename = part.get_filename()

                        self.log('info', "Switching session to {0}".format(filename))

                        if data:
                            tmp_path = os.path.join(tempfile.gettempdir(), filename)
                            with open(tmp_path, 'w') as tmp:
                                tmp.write(data)
                            __sessions__.new(tmp_path)
                            return

        def email_envelope(msg):
            # Envelope
            self.log('info', "Email envelope:")
            rows = [
                ['Subject', msg.get("Subject")],
                ['To', msg.get("To")],
                ['From', msg.get("From")],
                ['Cc', msg.get("Cc")],
                ['Bcc', msg.get("Bcc")],
                ['Date', msg.get("Date")]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_header(msg):
            # Headers
            rows = []
            for x in msg.keys():
                # Adding Received to ignore list. this has to be handeled separately if there are more then one line
                if x not in ['Subject', 'From', 'To', 'Date', 'Cc', 'Bcc', 'DKIM-Signature', 'Received']:
                    rows.append([x, string_clean(msg.get(x))])
            for x in msg.get_all('Received'):
                rows.append(['Received', string_clean(x)])
            self.log('info', "Email headers:")
            rows = sorted(rows, key=lambda entry: entry[0])
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_trace(msg, verbose):
            rows = []
            if verbose:
                fields = ['from', 'by', 'with', 'id', 'for', 'timestamp']
            else:
                fields = ['from', 'by', 'timestamp']
            for x in msg.get_all('Received'):
                x = string_clean(x)
                cre = re.compile("""
                    (?: from \s+ (?P<from>.*?) (?=by|with|id|ID|for|;|$) )?
                    (?: by \s+ (?P<by>.*?) (?=with|id|ID|for|;|$) )?
                    (?: with \s+ (?P<with>.*?) (?=id|ID|for|;|$) )?
                    (?: (id|ID) \s+ (?P<id>.*?) (?=for|;|$) )?
                    (?: for \s+ (?P<for>.*?) (?=;|$) )?
                    (?: \s* ; \s* (?P<timestamp>.*) )?
                    """, flags=re.X | re.I)
                m = cre.search(x)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                t = []
                for groupname in fields:
                    t.append(string_clean(m.group(groupname)))
                rows.insert(0, t)
            self.log('info', "Email path trace:")
            self.log('table', dict(header=fields, rows=rows))
            return

        def email_spoofcheck(msg, dnsenabled):
            self.log('info', "Email spoof check:")

            # test 1: check if From address is the same as Sender, Reply-To, and Return-Path
            rows = [
                ['Sender', string_clean(msg.get("Sender"))],
                ['From', string_clean(msg.get("From"))],
                ['Reply-To', string_clean(msg.get("Reply-To"))],
                ['Return-Path', string_clean(msg.get("Return-Path"))]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            addr = {
                'Sender': email.utils.parseaddr(string_clean(msg.get("Sender")))[1],
                'From': email.utils.parseaddr(string_clean(msg.get("From")))[1],
                'Reply-To': email.utils.parseaddr(string_clean(msg.get("Reply-To")))[1],
                'Return-Path': email.utils.parseaddr(string_clean(msg.get("Return-Path")))[1]
            }
            if (addr['From'] == ''):
                self.log('error', "No From address!")
                return
            elif addr['Sender'] and (addr['From'] != addr['Sender']):
                self.log('warning', "Email FAILED: From address different than Sender")
            elif addr['Reply-To'] and (addr['From'] != addr['Reply-To']):
                self.log('warning', "Email FAILED: From address different than Reply-To")
            elif addr['Return-Path'] and (addr['From'] != addr['Return-Path']):
                self.log('warning', "Email FAILED: From address different than Return-Path")
            else:
                self.log('success', "Email PASSED: From address the same as Sender, Reply-To, and Return-Path")

            # test 2: check to see if first Received: by domain matches sender MX domain
            if not dnsenabled:
                self.log('info', "Unable to run Received by / sender check without dnspython available")
            else:
                r = msg.get_all('Received')[-1]
                m = re.search("by\s+(\S*?)(?:\s+\(.*?\))?\s+with", r)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                byname = m.group(1)
                # this can be either a name or an IP
                m = re.search("(\w+\.\w+|\d+\.\d+\.\d+\.\d+)$", byname)
                if not m:
                    self.log('error', "Could not find domain or IP in Received by field")
                    return
                bydomain = m.group(1)
                domains = [['Received by', bydomain]]
                # if it's an IP, do the reverse lookup
                m = re.search("\.\d+$", bydomain)
                if m:
                    bydomain = str(dns.reversename.from_address(bydomain)).strip('.')
                    domains.append(['Received by reverse lookup', bydomain])
                # if the email has a Sender header, use that
                if (addr['Sender'] != ""):
                    m = re.search("(\w+\.\w+)$", addr['Sender'])
                    if not m:
                        self.log('error', "Sender header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['Sender', fromdomain])
                # otherwise, use the From header
                else:
                    m = re.search("(\w+\.\w+)$", addr['From'])
                    if not m:
                        self.log('error', "From header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['From', fromdomain])

                bymatch = False
                try:
                    mx = dns.resolver.query(fromdomain, 'MX')
                    if mx :
                        for rdata in mx:
                            m = re.search("(\w+\.\w+).$", str(rdata.exchange))
                            if not m:
                                self.log('error', "MX domain regex didn't match")
                                continue
                            domains.append(['MX for ' + fromdomain, m.group(1)])
                            if bydomain == m.group(1):
                                bymatch = True
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                except:
                    domains.append(['MX for ' + fromdomain, "not registered in DNS"])
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                if bymatch:
                    self.log('success', "Email PASSED: Received by domain found in Sender/From MX domains")
                else:
                    self.log('warning', "Email FAILED: Could not match Received by domain to Sender/From MX")

            # test 3: look at SPF records
            rspf = []
            results = set()
            allspf = msg.get_all('Received-SPF')
            if not allspf:
                return
            for spf in allspf:
                m = re.search("\s*(\w+)\s+\((.*?):\s*(.*?)\)\s+(.*);", string_clean(spf))
                if not m:
                    self.log('error', "Received-SPF regex didn't match")
                    return
                rspf.append([m.group(2), m.group(1), m.group(3), m.group(4)])
                results = results | {m.group(1)}
            self.log('table', dict(header=['Domain', 'Action', 'Info', 'Additional'], rows=rspf))
            if results & {'fail', 'softfail'}:
                self.log('warning', "Email FAILED: Found fail or softfail SPF results")
            elif results & {'none', 'neutral'}:
                self.log('warning', "Email NEUTRAL: Found none or neutral SPF results")
            elif results & {'permerror', 'temperror'}:
                self.log('warning', "Email NEUTRAL: Found error condition")
            elif results & {'pass'}:
                self.log('success', "Email PASSED: Found SPF pass result")

            return

        def email_attachments(msg, ole_flag):
            # Attachments
            att_count = 0
            rows = []
            links = []
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass

            else:
                # Walk through email string.
                for part in msg.walk():
                    content_type = part.get_content_type()

                    if content_type == 'multipart':
                        continue

                    if content_type in ('text/plain', 'text/html'):
                        part_content = part.get_payload(decode=True)
                        for link in re.findall(r'(https?://[^"<>\s]+)', part_content):
                            if link not in links:
                                links.append(link)

                    if content_type == 'message/rfc822':
                        part_content = part.as_string()
                        m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", part_content, flags=re.S)
                        if not m:
                            self.log('error', "Could not extract RFC822 formatted message")
                            return
                        part_content = m.group(1)
                        att_size = len(part_content)
                        att_file_name = "rfc822msg_{0}.eml".format(att_size)
                        att_md5 = hashlib.md5(part_content).hexdigest()
                        att_count += 1
                        rows.append([att_count, att_file_name, content_type, att_size, att_md5])
                        continue

                    if not part.get('Content-Disposition'):
                        # These are not attachments.
                        continue

                    att_file_name = part.get_filename()
                    att_size = len(part_content)

                    if not att_file_name:
                        continue

                    att_data = part.get_payload(decode=True)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    att_count += 1
                    rows.append([att_count, att_file_name, part.get_content_type(), att_size, att_md5])

            self.log('info', "Email attachments (total: {0}):".format(att_count))
            if att_count > 0:
                self.log('table', dict(header=['ID', 'FileName', 'Content Type', 'File Size', 'MD5'], rows=rows))

            self.log('info', "Email links:")
            for link in links:
                self.log('item', link)
            return

        # Start Here
        if not __sessions__.is_set():
            self.log('error', "No session opened")
            return

        super(EmailParse, self).run(*args)
        if self.args is None:
            return

        # see if we can load the dns library for MX lookup spoof detecton
        try:
            import dns.resolver
            import dns.reversename
            dnsenabled = True
        except ImportError:
            dnsenabled = False

        # Try to open as an ole msg, if not treat as email string
        try:
            ole = olefile.OleFileIO(__sessions__.current.file.path)
            ole_flag = True
        except:
            ole_flag = False
            email_handle = open(__sessions__.current.file.path)
            msg = email.message_from_file(email_handle)
            email_handle.close()

        if self.args.open is not None:
            if ole_flag:
                msg = ole
            att_session(self.args.open, msg, ole_flag)
        elif self.args.envelope:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
        elif self.args.attach:
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        elif self.args.header:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_header(msg)
        elif self.args.trace:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, False)
        elif self.args.traceall:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, True)
        elif self.args.spoofcheck:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_spoofcheck(msg, dnsenabled)
        elif self.args.all:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
            email_header(msg)
            email_trace(msg, True)
            email_spoofcheck(msg, dnsenabled)
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        else:
            self.log('error', 'At least one of the parameters is required')
            self.usage()


try:
            import dns.resolver
            import dns.reversename
            dnsenabled = True
        except ImportError:
            dnsenabled = False

# This file is part of Viper - http://example.com

import os
import re
import email
import hashlib
import tempfile
import olefile

from lib.common.abstracts import Module
from lib.core.session import __sessions__

class EmailParse(Module):
    cmd = 'email'
    description = 'Parse eml and msg email files'
    authors = ['Kevin Breen', 'nex']

    def __init__(self):
        super(EmailParse, self).__init__()
        self.parser.add_argument('-e', '--envelope', action='store_true', help='Show the email envelope')
        self.parser.add_argument('-f', '--attach', action='store_true', help='Show Attachment information')
        self.parser.add_argument('-r', '--header', action='store_true', help='Show email Header information')
        self.parser.add_argument('-t', '--trace', action='store_true', help='Show email path via Received headers')
        self.parser.add_argument('-T', '--traceall', action='store_true', help='Show email path via verbose Received headers')
        self.parser.add_argument('-s', '--spoofcheck', action='store_true', help='Test email for possible spoofing')
        self.parser.add_argument('-a', '--all', action='store_true', help='Run all the options')
        self.parser.add_argument('-o', '--open', type=int, help='Switch session to the specified attachment')

    def run(self, *args):

        def string_clean(value):
            if value:
                return re.sub('[\n\t\r]', '', value)
            return ""

        def parse_ole_msg(ole):
            stream_dirs = ole.listdir()
            for stream in stream_dirs:
                # get stream that contains the email header
                if stream[0].startswith('__substg1.0_007D'):
                    email_header = ole.openstream(stream).read()
                    if stream[0].endswith('001F'):  # Unicode probably needs something better than just stripping \x00
                        email_header = email_header.replace('\x00', '')
            # If it came from outlook we may need to trim some lines
            try:
                email_header = email_header.split('Version 2.0\x0d\x0a', 1)[1]
            except:
                pass

            # Leaving us an RFC compliant email to parse
            msg = email.message_from_string(email_header)
            return msg

        def parse_ole_attachments(ole):
            # Hard part now, each part of the attachment is in a seperate stream

            # need to get a unique stream id for each att
            # its in the streamname as an 8 digit number.
            for i in range(20):  # arbitrary count of emails. i dont expecet this many
                stream_number = str(i).zfill(8)
                stream_name = '__attach_version1.0_#' + stream_number
                # Unicode
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass
                # ASCII
                try:
                    att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                    att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                    att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    att_size = len(att_data)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    print i, att_size, att_md5, att_filename, att_mime
                except:
                    pass

        def att_session(att_id, msg, ole_flag):
            att_count = 0
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_filename = att_filename.replace('\x00', '')
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                    except:
                        pass
                    if i == att_id:
                        self.log('info', "Switching session to {0}".format(att_filename))
                        tmp_path = os.path.join(tempfile.gettempdir(), att_filename)
                        with open(tmp_path, 'w') as tmp:
                            tmp.write(att_data)
                        __sessions__.new(tmp_path)
                        return

            else:
                for part in msg.walk():
                    if part.get_content_type() == 'message/rfc822':
                        rfc822 = True
                    else:
                        rfc822 = False

                    if part.get_content_maintype() == 'multipart' \
                        or not part.get('Content-Disposition') \
                            and not rfc822:
                        continue

                    att_count += 1
                    if att_count == att_id:
                        if rfc822:
                            data = part.as_string()
                            m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", data, flags=re.S)
                            if not m:
                                self.log('error', "Could not extract RFC822 formatted message")
                                return
                            data = m.group(1)
                            att_size = len(data)
                            filename = "rfc822msg_{0}.eml".format(att_size)
                        else:
                            data = part.get_payload(decode=True)
                            filename = part.get_filename()

                        self.log('info', "Switching session to {0}".format(filename))

                        if data:
                            tmp_path = os.path.join(tempfile.gettempdir(), filename)
                            with open(tmp_path, 'w') as tmp:
                                tmp.write(data)
                            __sessions__.new(tmp_path)
                            return

        def email_envelope(msg):
            # Envelope
            self.log('info', "Email envelope:")
            rows = [
                ['Subject', msg.get("Subject")],
                ['To', msg.get("To")],
                ['From', msg.get("From")],
                ['Cc', msg.get("Cc")],
                ['Bcc', msg.get("Bcc")],
                ['Date', msg.get("Date")]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_header(msg):
            # Headers
            rows = []
            for x in msg.keys():
                # Adding Received to ignore list. this has to be handeled separately if there are more then one line
                if x not in ['Subject', 'From', 'To', 'Date', 'Cc', 'Bcc', 'DKIM-Signature', 'Received']:
                    rows.append([x, string_clean(msg.get(x))])
            for x in msg.get_all('Received'):
                rows.append(['Received', string_clean(x)])
            self.log('info', "Email headers:")
            rows = sorted(rows, key=lambda entry: entry[0])
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            return

        def email_trace(msg, verbose):
            rows = []
            if verbose:
                fields = ['from', 'by', 'with', 'id', 'for', 'timestamp']
            else:
                fields = ['from', 'by', 'timestamp']
            for x in msg.get_all('Received'):
                x = string_clean(x)
                cre = re.compile("""
                    (?: from \s+ (?P<from>.*?) (?=by|with|id|ID|for|;|$) )?
                    (?: by \s+ (?P<by>.*?) (?=with|id|ID|for|;|$) )?
                    (?: with \s+ (?P<with>.*?) (?=id|ID|for|;|$) )?
                    (?: (id|ID) \s+ (?P<id>.*?) (?=for|;|$) )?
                    (?: for \s+ (?P<for>.*?) (?=;|$) )?
                    (?: \s* ; \s* (?P<timestamp>.*) )?
                    """, flags=re.X | re.I)
                m = cre.search(x)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                t = []
                for groupname in fields:
                    t.append(string_clean(m.group(groupname)))
                rows.insert(0, t)
            self.log('info', "Email path trace:")
            self.log('table', dict(header=fields, rows=rows))
            return

        def email_spoofcheck(msg, dnsenabled):
            self.log('info', "Email spoof check:")

            # test 1: check if From address is the same as Sender, Reply-To, and Return-Path
            rows = [
                ['Sender', string_clean(msg.get("Sender"))],
                ['From', string_clean(msg.get("From"))],
                ['Reply-To', string_clean(msg.get("Reply-To"))],
                ['Return-Path', string_clean(msg.get("Return-Path"))]
            ]
            self.log('table', dict(header=['Key', 'Value'], rows=rows))
            addr = {
                'Sender': email.utils.parseaddr(string_clean(msg.get("Sender")))[1],
                'From': email.utils.parseaddr(string_clean(msg.get("From")))[1],
                'Reply-To': email.utils.parseaddr(string_clean(msg.get("Reply-To")))[1],
                'Return-Path': email.utils.parseaddr(string_clean(msg.get("Return-Path")))[1]
            }
            if (addr['From'] == ''):
                self.log('error', "No From address!")
                return
            elif addr['Sender'] and (addr['From'] != addr['Sender']):
                self.log('warning', "Email FAILED: From address different than Sender")
            elif addr['Reply-To'] and (addr['From'] != addr['Reply-To']):
                self.log('warning', "Email FAILED: From address different than Reply-To")
            elif addr['Return-Path'] and (addr['From'] != addr['Return-Path']):
                self.log('warning', "Email FAILED: From address different than Return-Path")
            else:
                self.log('success', "Email PASSED: From address the same as Sender, Reply-To, and Return-Path")

            # test 2: check to see if first Received: by domain matches sender MX domain
            if not dnsenabled:
                self.log('info', "Unable to run Received by / sender check without dnspython available")
            else:
                r = msg.get_all('Received')[-1]
                m = re.search("by\s+(\S*?)(?:\s+\(.*?\))?\s+with", r)
                if not m:
                    self.log('error', "Received header regex didn't match")
                    return
                byname = m.group(1)
                # this can be either a name or an IP
                m = re.search("(\w+\.\w+|\d+\.\d+\.\d+\.\d+)$", byname)
                if not m:
                    self.log('error', "Could not find domain or IP in Received by field")
                    return
                bydomain = m.group(1)
                domains = [['Received by', bydomain]]
                # if it's an IP, do the reverse lookup
                m = re.search("\.\d+$", bydomain)
                if m:
                    bydomain = str(dns.reversename.from_address(bydomain)).strip('.')
                    domains.append(['Received by reverse lookup', bydomain])
                # if the email has a Sender header, use that
                if (addr['Sender'] != ""):
                    m = re.search("(\w+\.\w+)$", addr['Sender'])
                    if not m:
                        self.log('error', "Sender header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['Sender', fromdomain])
                # otherwise, use the From header
                else:
                    m = re.search("(\w+\.\w+)$", addr['From'])
                    if not m:
                        self.log('error', "From header regex didn't match")
                        return
                    fromdomain = m.group(1)
                    domains.append(['From', fromdomain])

                bymatch = False
                try:
                    mx = dns.resolver.query(fromdomain, 'MX')
                    if mx :
                        for rdata in mx:
                            m = re.search("(\w+\.\w+).$", str(rdata.exchange))
                            if not m:
                                self.log('error', "MX domain regex didn't match")
                                continue
                            domains.append(['MX for ' + fromdomain, m.group(1)])
                            if bydomain == m.group(1):
                                bymatch = True
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                except:
                    domains.append(['MX for ' + fromdomain, "not registered in DNS"])
                    self.log('table', dict(header=['Key', 'Value'], rows=domains))
                if bymatch:
                    self.log('success', "Email PASSED: Received by domain found in Sender/From MX domains")
                else:
                    self.log('warning', "Email FAILED: Could not match Received by domain to Sender/From MX")

            # test 3: look at SPF records
            rspf = []
            results = set()
            allspf = msg.get_all('Received-SPF')
            if not allspf:
                return
            for spf in allspf:
                m = re.search("\s*(\w+)\s+\((.*?):\s*(.*?)\)\s+(.*);", string_clean(spf))
                if not m:
                    self.log('error', "Received-SPF regex didn't match")
                    return
                rspf.append([m.group(2), m.group(1), m.group(3), m.group(4)])
                results = results | {m.group(1)}
            self.log('table', dict(header=['Domain', 'Action', 'Info', 'Additional'], rows=rspf))
            if results & {'fail', 'softfail'}:
                self.log('warning', "Email FAILED: Found fail or softfail SPF results")
            elif results & {'none', 'neutral'}:
                self.log('warning', "Email NEUTRAL: Found none or neutral SPF results")
            elif results & {'permerror', 'temperror'}:
                self.log('warning', "Email NEUTRAL: Found error condition")
            elif results & {'pass'}:
                self.log('success', "Email PASSED: Found SPF pass result")

            return

        def email_attachments(msg, ole_flag):
            # Attachments
            att_count = 0
            rows = []
            links = []
            if ole_flag:
                ole = msg
                # Hard part now, each part of the attachment is in a seperate stream

                # need to get a unique stream id for each att
                # its in the streamname as an 8 digit number.
                for i in range(20):  # arbitrary count of emails. i dont expecet this many
                    stream_number = str(i).zfill(8)
                    stream_name = '__attach_version1.0_#' + stream_number
                    # Unicode
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001F').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001F').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass
                    # ASCII
                    try:
                        att_filename = ole.openstream(stream_name + '/__substg1.0_3704001E').read()
                        att_mime = ole.openstream(stream_name + '/__substg1.0_370E001E').read()
                        att_data = ole.openstream(stream_name + '/__substg1.0_37010102').read()
                        att_size = len(att_data)
                        att_md5 = hashlib.md5(att_data).hexdigest()
                        rows.append([i, att_filename, att_mime, att_size, att_md5])
                        att_count += 1
                    except:
                        pass

            else:
                # Walk through email string.
                for part in msg.walk():
                    content_type = part.get_content_type()

                    if content_type == 'multipart':
                        continue

                    if content_type in ('text/plain', 'text/html'):
                        part_content = part.get_payload(decode=True)
                        for link in re.findall(r'(https?://[^"<>\s]+)', part_content):
                            if link not in links:
                                links.append(link)

                    if content_type == 'message/rfc822':
                        part_content = part.as_string()
                        m = re.match("Content-Type: message/rfc822\r?\n\r?\n(.*)", part_content, flags=re.S)
                        if not m:
                            self.log('error', "Could not extract RFC822 formatted message")
                            return
                        part_content = m.group(1)
                        att_size = len(part_content)
                        att_file_name = "rfc822msg_{0}.eml".format(att_size)
                        att_md5 = hashlib.md5(part_content).hexdigest()
                        att_count += 1
                        rows.append([att_count, att_file_name, content_type, att_size, att_md5])
                        continue

                    if not part.get('Content-Disposition'):
                        # These are not attachments.
                        continue

                    att_file_name = part.get_filename()
                    att_size = len(part_content)

                    if not att_file_name:
                        continue

                    att_data = part.get_payload(decode=True)
                    att_md5 = hashlib.md5(att_data).hexdigest()
                    att_count += 1
                    rows.append([att_count, att_file_name, part.get_content_type(), att_size, att_md5])

            self.log('info', "Email attachments (total: {0}):".format(att_count))
            if att_count > 0:
                self.log('table', dict(header=['ID', 'FileName', 'Content Type', 'File Size', 'MD5'], rows=rows))

            self.log('info', "Email links:")
            for link in links:
                self.log('item', link)
            return

        # Start Here
        if not __sessions__.is_set():
            self.log('error', "No session opened")
            return

        super(EmailParse, self).run(*args)
        if self.args is None:
            return

        # see if we can load the dns library for MX lookup spoof detecton

        # Try to open as an ole msg, if not treat as email string
        try:
            ole = olefile.OleFileIO(__sessions__.current.file.path)
            ole_flag = True
        except:
            ole_flag = False
            email_handle = open(__sessions__.current.file.path)
            msg = email.message_from_file(email_handle)
            email_handle.close()

        if self.args.open is not None:
            if ole_flag:
                msg = ole
            att_session(self.args.open, msg, ole_flag)
        elif self.args.envelope:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
        elif self.args.attach:
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        elif self.args.header:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_header(msg)
        elif self.args.trace:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, False)
        elif self.args.traceall:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_trace(msg, True)
        elif self.args.spoofcheck:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_spoofcheck(msg, dnsenabled)
        elif self.args.all:
            if ole_flag:
                msg = parse_ole_msg(ole)
            email_envelope(msg)
            email_header(msg)
            email_trace(msg, True)
            email_spoofcheck(msg, dnsenabled)
            if ole_flag:
                msg = ole
            email_attachments(msg, ole_flag)
        else:
            self.log('error', 'At least one of the parameters is required')
            self.usage()


from pisi.actionsapi import qt4
from pisi.actionsapi import pisitools
from pisi.actionsapi import shelltools

def setup():
    pisitools.dosed("resources.qrc", ".*logo-.*$")
    qt4.configure("clipgrab.pro")

qt4.make()
    shelltools.chmod("icon.png", 0644)

def install():
    pisitools.dobin("clipgrab")
    pisitools.insinto("/usr/share/pixmaps/", "icon.png", "clipgrab.png")
    pisitools.dodoc("COPYING", "license.odt")


pisitools.dobin("clipgrab")
    pisitools.insinto("/usr/share/pixmaps/", "icon.png", "clipgrab.png")
    pisitools.dodoc("COPYING", "license.odt")

from pisi.actionsapi import qt4
from pisi.actionsapi import pisitools
from pisi.actionsapi import shelltools

def setup():
    pisitools.dosed("resources.qrc", ".*logo-.*$")
    qt4.configure("clipgrab.pro")

def build():
    qt4.make()
    shelltools.chmod("icon.png", 0644)

def install():


from pisi.actionsapi import qt4
from pisi.actionsapi import pisitools
from pisi.actionsapi import shelltools

def setup():
    pisitools.dosed("resources.qrc", ".*logo-.*$")
    qt4.configure("clipgrab.pro")

def build():
    qt4.make()
    shelltools.chmod("icon.png", 0644)

def install():
    pisitools.dobin("clipgrab")
    pisitools.insinto("/


from beckett.resources import BaseResource

class PokemonResource(BaseResource):
    class Meta(BaseResource.Meta):
        name = 'Pokemon'
        resource_name = 'pokemon'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'national_id',
            'abilities',
            'egg_groups',
            'evolutions',
            'descriptions',
            'moves',
            'types',
            'catch_rate',
            'species',
            'hp',
            'attack',
            'defense',
            'name',
            'sp_atk',
            'sp_def',
            'speed',
            'total',
            'egg_cycles',
            'ev_yield',
            'exp',
            'growth_rate',
            'height',
            'weight',
            'happiness',
            'male_female_ratio',
            'sprites',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class MoveResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Move'
        resource_name = 'move'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'accuracy',
            'category',
            'power',
            'pp',
            'name',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class TypeResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Type'
        resource_name = 'type'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'ineffective',
            'resistance',
            'super_effective',
            'weakness',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class AbilityResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Ability'
        resource_name = 'ability'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
        )

    @staticmethod

return '{}/{}/'.format(url, uid)

class EggResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Egg'
        resource_name = 'egg'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class DescriptionResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Description'
        resource_name = 'description'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
            'pokemon',
            'games',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class SpriteResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Sprite'
        resource_name = 'sprite'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
            'image',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class GameResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Game'
        resource_name = 'game'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'generation',
            'release_year',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)


from beckett.resources import BaseResource

class PokemonResource(BaseResource):
    class Meta(BaseResource.Meta):
        name = 'Pokemon'
        resource_name = 'pokemon'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'national_id',
            'abilities',
            'egg_groups',
            'evolutions',
            'descriptions',
            'moves',
            'types',
            'catch_rate',
            'species',
            'hp',
            'attack',
            'defense',
            'name',
            'sp_atk',
            'sp_def',
            'speed',
            'total',
            'egg_cycles',
            'ev_yield',
            'exp',
            'growth_rate',
            'height',
            'weight',
            'happiness',
            'male_female_ratio',
            'sprites',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class MoveResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Move'
        resource_name = 'move'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'accuracy',
            'category',
            'power',
            'pp',
            'name',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class TypeResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Type'
        resource_name = 'type'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'ineffective',
            'resistance',
            'super_effective',
            'weakness',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class AbilityResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Ability'
        resource_name = 'ability'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class EggResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Egg'
        resource_name = 'egg'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class DescriptionResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Description'
        resource_name = 'description'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
            'pokemon',
            'games',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class SpriteResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Sprite'
        resource_name = 'sprite'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
            'image',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class GameResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Game'
        resource_name = 'game'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'generation',
            'release_year',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!


from beckett.resources import BaseResource

class PokemonResource(BaseResource):
    class Meta(BaseResource.Meta):
        name = 'Pokemon'
        resource_name = 'pokemon'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'national_id',
            'abilities',
            'egg_groups',
            'evolutions',
            'descriptions',
            'moves',
            'types',
            'catch_rate',
            'species',
            'hp',
            'attack',
            'defense',
            'name',
            'sp_atk',
            'sp_def',
            'speed',
            'total',
            'egg_cycles',
            'ev_yield',
            'exp',
            'growth_rate',
            'height',
            'weight',
            'happiness',
            'male_female_ratio',
            'sprites',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class MoveResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Move'
        resource_name = 'move'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'accuracy',
            'category',
            'power',
            'pp',
            'name',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class TypeResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Type'
        resource_name = 'type'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'ineffective',
            'resistance',
            'super_effective',
            'weakness',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '

class AbilityResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Ability'
        resource_name = 'ability'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class EggResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Egg'
        resource_name = 'egg'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class DescriptionResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Description'
        resource_name = 'description'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'description',
            'pokemon',
            'games',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class SpriteResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Sprite'
        resource_name = 'sprite'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'pokemon',
            'image',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)

class GameResource(BaseResource):

    class Meta(BaseResource.Meta):
        name = 'Game'
        resource_name = 'game'
        identifier = 'id'
        methods = (
            'get',
        )
        attributes = (
            'created',
            'modified',
            'id',
            'name',
            'generation',
            'release_year',
        )

    @staticmethod
    def get_single_resource_url(url, uid, **kwargs):
        # Needs a slash on the end!
        return '{}/{}/'.format(url, uid)


# Generated from BaserowFormula.g4 by ANTLR 4.8
from antlr4 import *
if __name__ is not None and "." in __name__:
    from .BaserowFormula import BaserowFormula
else:
    from BaserowFormula import BaserowFormula

# This class defines a complete generic visitor for a parse tree produced by BaserowFormula.

class BaserowFormulaVisitor(ParseTreeVisitor):

    # Visit a parse tree produced by BaserowFormula#root.
    def visitRoot(self, ctx:BaserowFormula.RootContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldReference.
    def visitFieldReference(self, ctx:BaserowFormula.FieldReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#StringLiteral.
    def visitStringLiteral(self, ctx:BaserowFormula.StringLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#Brackets.
    def visitBrackets(self, ctx:BaserowFormula.BracketsContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BooleanLiteral.
    def visitBooleanLiteral(self, ctx:BaserowFormula.BooleanLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#DecimalLiteral.
    def visitDecimalLiteral(self, ctx:BaserowFormula.DecimalLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FunctionCall.
    def visitFunctionCall(self, ctx:BaserowFormula.FunctionCallContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldByIdReference.
    def visitFieldByIdReference(self, ctx:BaserowFormula.FieldByIdReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#IntegerLiteral.
    def visitIntegerLiteral(self, ctx:BaserowFormula.IntegerLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BinaryOp.
    def visitBinaryOp(self, ctx:BaserowFormula.BinaryOpContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#func_name.
    def visitFunc_name(self, ctx:BaserowFormula.Func_nameContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#field_reference.

return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#identifier.
    def visitIdentifier(self, ctx:BaserowFormula.IdentifierContext):
        return self.visitChildren(ctx)

del BaserowFormula


# Generated from BaserowFormula.g4 by ANTLR 4.8
from antlr4 import *
if __name__ is not None and "." in __name__:
    from .BaserowFormula import BaserowFormula
else:
    from BaserowFormula import BaserowFormula

# This class defines a complete generic visitor for a parse tree produced by BaserowFormula.

class BaserowFormulaVisitor(ParseTreeVisitor):

    # Visit a parse tree produced by BaserowFormula#root.
    def visitRoot(self, ctx:BaserowFormula.RootContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldReference.
    def visitFieldReference(self, ctx:BaserowFormula.FieldReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#StringLiteral.
    def visitStringLiteral(self, ctx:BaserowFormula.StringLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#Brackets.
    def visitBrackets(self, ctx:BaserowFormula.BracketsContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BooleanLiteral.
    def visitBooleanLiteral(self, ctx:BaserowFormula.BooleanLiteralContext):

    # Visit a parse tree produced by BaserowFormula#DecimalLiteral.
    def visitDecimalLiteral(self, ctx:BaserowFormula.DecimalLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FunctionCall.
    def visitFunctionCall(self, ctx:BaserowFormula.FunctionCallContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldByIdReference.
    def visitFieldByIdReference(self, ctx:BaserowFormula.FieldByIdReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#IntegerLiteral.
    def visitIntegerLiteral(self, ctx:BaserowFormula.IntegerLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BinaryOp.
    def visitBinaryOp(self, ctx:BaserowFormula.BinaryOpContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#func_name.
    def visitFunc_name(self, ctx:BaserowFormula.Func_nameContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#field_reference.
    def visitField_reference(self, ctx:BaserowFormula.Field_referenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#identifier.
    def visitIdentifier(self, ctx:BaserowFormula.IdentifierContext):
        return self.visitChildren(ctx)

del BaserowFormula


# Generated from BaserowFormula.g4 by ANTLR 4.8
from antlr4 import *
if __name__ is not None and "." in __name__:
    from .BaserowFormula import BaserowFormula
else:
    from BaserowFormula import BaserowFormula

# This class defines a complete generic visitor for a parse tree produced by BaserowFormula.

class BaserowFormulaVisitor(ParseTreeVisitor):

    # Visit a parse tree produced by BaserowFormula#root.
    def visitRoot(self, ctx:BaserowFormula.RootContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldReference.
    def visitFieldReference(self, ctx:BaserowFormula.FieldReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#StringLiteral.
    def visitStringLiteral(self, ctx:BaserowFormula.StringLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#Brackets.
    def visitBrackets(self, ctx:BaserowFormula.BracketsContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BooleanLiteral.
    def visitBooleanLiteral(self, ctx:BaserowFormula.BooleanLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#DecimalLiteral.
    def visitDecimalLiteral(self, ctx:BaserowFormula.DecimalLiteralContext):
        return self.visitChildren(ct

    # Visit a parse tree produced by BaserowFormula#FunctionCall.
    def visitFunctionCall(self, ctx:BaserowFormula.FunctionCallContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#FieldByIdReference.
    def visitFieldByIdReference(self, ctx:BaserowFormula.FieldByIdReferenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#IntegerLiteral.
    def visitIntegerLiteral(self, ctx:BaserowFormula.IntegerLiteralContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#BinaryOp.
    def visitBinaryOp(self, ctx:BaserowFormula.BinaryOpContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#func_name.
    def visitFunc_name(self, ctx:BaserowFormula.Func_nameContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#field_reference.
    def visitField_reference(self, ctx:BaserowFormula.Field_referenceContext):
        return self.visitChildren(ctx)

    # Visit a parse tree produced by BaserowFormula#identifier.
    def visitIdentifier(self, ctx:BaserowFormula.IdentifierContext):
        return self.visitChildren(ctx)

del BaserowFormula


from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    try_get,
    unescapeHTML,
    url_or_none,
    urljoin,
)

class WWEBaseIE(InfoExtractor):
    _SUBTITLE_LANGS = {
        'English': 'en',
        'Deutsch': 'de',
    }

    def _extract_entry(self, data, url, video_id=None):
        video_id = compat_str(video_id or data['nid'])
        title = data['title']

        formats = self._extract_m3u8_formats(
            data['file'], video_id, 'mp4', entry_protocol='m3u8_native',
            m3u8_id='hls')

        description = data.get('description')
        thumbnail = urljoin(url, data.get('image'))
        series = data.get('show_name')
        episode = data.get('episode_name')

        subtitles = {}
        tracks = data.get('tracks')
        if isinstance(tracks, list):
            for track in tracks:
                if not isinstance(track, dict):
                    continue
                if track.get('kind') != 'captions':
                    continue
                track_file = url_or_none(track.get('file'))
                if not track_file:
                    continue
                label = track.get('label')
                lang = self._SUBTITLE_LANGS.get(label, label) or 'en'
                subtitles.setdefault(lang, []).append({
                    'url': track_file,
                })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'series': series,
            'episode': episode,
            'formats': formats,
            'subtitles': subtitles,
        }

class WWEIE(WWEBaseIE):
    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*videos/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://example.com',
        'md5': '92811c6a14bfc206f7a6a9c5d9140184',
        'info_dict': {
            'id': '40048199',
            'ext': 'mp4',
            'title': 'Daniel Bryan vs. Andrade "Cien" Almas: SmackDown LIVE, Sept. 4, 2018',
            'description': 'md5:2d7424dbc6755c61a0e649d2a8677f67',
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }]

display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        landing = self._parse_json(
            self._html_search_regex(
                r'(?s)Drupal\.settings\s*,\s*({.+?})\s*\)\s*;',
                webpage, 'drupal settings'),
            display_id)['WWEVideoLanding']

        data = landing['initialVideo']['playlist'][0]
        video_id = landing.get('initialVideoId')

        info = self._extract_entry(data, url, video_id)
        info['display_id'] = display_id
        return info

class WWEPlaylistIE(WWEBaseIE):
    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://example.com',
        'info_dict': {
            'id': '2018-11-12',
        },
        'playlist_mincount': 11,
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if WWEIE.suitable(url) else super(WWEPlaylistIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        entries = []
        for mobj in re.finditer(
                r'data-video\s*=\s*(["\'])(?P<data>{.+?})\1', webpage):
            video = self._parse_json(
                mobj.group('data'), display_id, transform_source=unescapeHTML,
                fatal=False)
            if not video:
                continue
            data = try_get(video, lambda x: x['playlist'][0], dict)
            if not data:
                continue
            try:
                entry = self._extract_entry(data, url)
            except Exception:
                continue
            entry['extractor_key'] = WWEIE.ie_key()
            entries.append(entry)

        return self.playlist_result(entries, display_id)


return False if WWEIE.suitable(url) else super(WWEPlaylistIE, cls).suitable(url)

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    try_get,
    unescapeHTML,
    url_or_none,
    urljoin,
)

class WWEBaseIE(InfoExtractor):
    _SUBTITLE_LANGS = {
        'English': 'en',
        'Deutsch': 'de',
    }

    def _extract_entry(self, data, url, video_id=None):
        video_id = compat_str(video_id or data['nid'])
        title = data['title']

        formats = self._extract_m3u8_formats(
            data['file'], video_id, 'mp4', entry_protocol='m3u8_native',
            m3u8_id='hls')

        description = data.get('description')
        thumbnail = urljoin(url, data.get('image'))
        series = data.get('show_name')
        episode = data.get('episode_name')

        subtitles = {}
        tracks = data.get('tracks')
        if isinstance(tracks, list):
            for track in tracks:
                if not isinstance(track, dict):
                    continue
                if track.get('kind') != 'captions':
                    continue
                track_file = url_or_none(track.get('file'))
                if not track_file:
                    continue
                label = track.get('label')
                lang = self._SUBTITLE_LANGS.get(label, label) or 'en'
                subtitles.setdefault(lang, []).append({
                    'url': track_file,
                })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'series': series,
            'episode': episode,
            'formats': formats,
            'subtitles': subtitles,
        }

class WWEIE(WWEBaseIE):
    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*videos/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://example.com',
        'md5': '92811c6a14bfc206f7a6a9c5d9140184',
        'info_dict': {
            'id': '40048199',
            'ext': 'mp4',
            'title': 'Daniel Bryan vs. Andrade "Cien" Almas: SmackDown LIVE, Sept. 4, 2018',
            'description': 'md5:2d7424dbc6755c61a0e649d2a8677f67',
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        landing = self._parse_json(
            self._html_search_regex(
                r'(?s)Drupal\.settings\s*,\s*({.+?})\s*\)\s*;',
                webpage, 'drupal settings'),
            display_id)['WWEVideoLanding']

        data = landing['initialVideo']['playlist'][0]
        video_id = landing.get('initialVideoId')

        info = self._extract_entry(data, url, video_id)
        info['display_id'] = display_id
        return info

class WWEPlaylistIE(WWEBaseIE):
    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://example.com',
        'info_dict': {
            'id': '2018-11-12',
        },
        'playlist_mincount': 11,
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }, {
        'url': 'http://example.com',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        entries = []
        for mobj in re.finditer(
                r'data-video\s*=\s*(["\'])(?P<data>{.+?})\1', webpage):
            video = self._parse_json(
                mobj.group('data'), display_id, transform_source=unescapeHTML,
                fatal=False)
            if not video:
                continue
            data = try_get(video, lambda x: x['playlist'][0], dict)
            if not data:
                continue
            try:
                entry = self._extract_entry(data, url)
            except Exception:
                continue
            entry['extractor_key'] = WWEIE.ie_key()
            entries.append(entry)

        return self.playlist_result(entries, display_id)