def file_exists(dbx, file_path, dest_path):

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):

    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")

    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


# add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to


import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_se

    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()

import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""

        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Local hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


import hashlib
import logging
import os

import dropbox
import retrace

LOG = logging.getLogger(__name__)
CHUNK_SIZE = 4 * 1024 * 1024

@retrace.retry(limit=10)
def upload_file(dbx, file_path, dest_path):

    f = open(file_path, "rb")
    file_size = os.path.getsize(file_path)
    if file_size <= CHUNK_SIZE:
        return dbx.files_upload(f.read(), dest_path)

    upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
    cursor = dropbox.files.UploadSessionCursor(
        session_id=upload_session_start_result.session_id, offset=f.tell()
    )
    commit = dropbox.files.CommitInfo(path=dest_path)
    prev = None
    while f.tell() < file_size:
        percentage = round((f.tell() / file_size) * 100)

        if prev is None or percentage > prev + 4:
            LOG.info(f"{percentage:3} %")
            prev = percentage

        if (file_size - f.tell()) <= CHUNK_SIZE:
            dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
        else:
            try:
                dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
                cursor.offset = f.tell()
            except dropbox.exceptions.ApiError as ex:
                if ex.error.is_incorrect_offset():
                    correct_offset = ex.error.get_incorrect_offset().correct_offset
                    cursor.offset = correct_offset
                    f.seek(correct_offset)

    LOG.info("100 %")

def compute_dropbox_hash(filename):

    with open(filename, "rb") as f:
        block_hashes = b""
        while True:
            chunk = f.read(CHUNK_SIZE)
            if not chunk:
                break
            block_hashes += hashlib.sha256(chunk).digest()
        return hashlib.sha256(block_hashes).hexdigest()

def file_exists(dbx, file_path, dest_path):
    try:
        metadata = dbx.files_get_metadata(dest_path)
    except Exception:
        LOG.info("No existing snapshot in dropox with this name")
        return False

    dropbox_hash = metadata.content_hash
    local_hash = compute_dropbox_hash(file_path)
    LOG.debug(f"Dropbox hash: {dropbox_hash}")
    LOG.debug(f"Loc

al hash: {local_hash}")
    if local_hash == dropbox_hash:
        return True

    # If the hash doesn't match, delete the file so we can re-upload it.
    # We might want to make this optional? a safer mode might be to
    # add a suffix?
    LOG.warn(
        "The snapshot conflicts with a file name in dropbox, the contents "
        "are different. The dropbox file will be deleted and replaced. "
        "Local hash: %s, Dropbox hash: %s",
        local_hash,
        dropbox_hash,
    )
    dbx.files_delete(dest_path)
    return False


# This file is part of the SPORCO package. Details of the copyright
# and user license can be found in the 'LICENSE.txt' file distributed

"""
Constained MOD
==============

This example demonstrates the use of :class:`.cmod.CnstrMOD` for computing a dictionary update via a constrained variant of the method of optimal directions :cite:`engan-1999-method`. This problem is mainly useful as a component within dictionary learning, but its use is demonstrated here since a user may wish to construct such objects as part of a custom dictionary learning algorithm, using :class:`.dictlrn.DictLearn`.
"""

from __future__ import print_function
from builtins import input
from builtins import range

import numpy as np

from sporco.admm import bpdn
from sporco.admm import cmod
from sporco import util
from sporco import plot

"""
Load training images.
"""

exim = util.ExampleImages(scaled=True, zoom=0.25, gray=True)
S1 = exim.image('barbara.png', idxexp=np.s_[10:522, 100:612])
S2 = exim.image('kodim23.png', idxexp=np.s_[:, 60:572])
S3 = exim.image('monarch.png', idxexp=np.s_[:, 160:672])
S4 = exim.image('sail.png', idxexp=np.s_[:, 210:722])
S5 = exim.image('tulips.png', idxexp=np.s_[:, 30:542])

"""
Extract all 8x8 image blocks, reshape, and subtract block means.
"""

S = util.imageblocks((S1, S2, S3, S4, S5), (8, 8))
S = np.reshape(S, (np.prod(S.shape[0:2]), S.shape[2]))
S -= np.mean(S, axis=0)

"""
Load initial dictionary.
"""

D0 = util.convdicts()['G:8x8x64']
D0 = np.reshape(D0, (np.prod(D0.shape[0:2]), D0.shape[2]))

"""
Compute sparse representation on current dictionary.
"""

lmbda = 0.1
opt = bpdn.BPDN.Options({'Verbose': True, 'MaxMainIter': 200,
                         'RelStopTol': 1e-3})
b = bpdn.BPDN(D0, S, lmbda, opt)
X = b.solve()

"""
Update dictionary for training image set.
"""

opt = cmod.CnstrMOD.Options({'Verbose': True, 'MaxMainIter': 100,
                             'RelStopTol': 1e-3, 'rho': 4e2})
c = cmod.CnstrMOD(X, S, None, opt)
D1 = c.solve()
print("CMOD solve time: %.2fs" % c.timer.elapsed('solve'))

"""
Display initial and final dictionaries.
"""

D0 = D0.reshape((8, 8,

 D0.shape[-1]))
D1 = D1.reshape((8, 8, D1.shape[-1]))
fig = plot.figure(figsize=(14, 7))
plot.subplot(1, 2, 1)
plot.imview(util.tiledict(D0), fig=fig, title='D0')
plot.subplot(1, 2, 2)
plot.imview(util.tiledict(D1), fig=fig, title='D1')
fig.show()

"""
Get iterations statistics from CMOD solver object and plot functional value, ADMM primary and dual residuals, and automatically adjusted ADMM penalty parameter against the iteration number.
"""

its = c.getitstat()
fig = plot.figure(figsize=(20, 5))
plot.subplot(1, 3, 1)
plot.plot(its.DFid, fig=fig, xlbl='Iterations', ylbl='Functional')
plot.subplot(1, 3, 2)
plot.plot(np.vstack((its.PrimalRsdl, its.DualRsdl)).T, fig=fig,
          ptyp='semilogy', xlbl='Iterations', ylbl='Residual',
          lgnd=['Primal', 'Dual'])
plot.subplot(1, 3, 3)
plot.plot(its.Rho, fig=fig, xlbl='Iterations', ylbl='Penalty Parameter')
fig.show()

# Wait for enter on keyboard
input()


"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

    def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

    r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

    def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

    r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):

root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):

def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

    r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

    def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")
        if head is not None:
            r.append("
{0}
\n".format(head.text))

        for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(d

cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

    def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

    r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))


if head is not None:
            r.append("<h2>{0}</h2>\n".format(head.text))

"""Content transformer

This module generates HTML data from LDOCE5's XML documents.
"""

from html import escape

import lxml.etree as et

from ..utils.text import enc_utf8
from .transform_body import _trans_assets, body2html
from .utils import shorten_id

def _get_text_r(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tag != "span":
            r.append("<{0}>{1}</{0}>".format(c.tag, _get_text_r(c)))
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _get_text_nr(e):
    r = []
    if e.text is not None:
        r.append(e.text)

    for c in e:
        if c.tail is not None:
            r.append(c.tail)

    return "".join(r).strip()

def _build_header(resnames, title=None, meta={}):
    r = ['<!DOCTYPE html>\n\n\n' '<meta charset="utf-8">\n']

    for k in meta:
        v = meta[k]
        r.append('<meta name="{0}" content="{1}" />'.format(escape(k), escape(v)))

    r.append(
        '\n'
    )
    r.append(
        '\n'
    )
    r.append(
        '<link href="static:///scripts/colorbox/colorbox.css" '
        'rel="stylesheet" type="text/css">\n'
    )
    for name in resnames:
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '<link href="static:///styles/{0}.css" '
            'rel="stylesheet" type="text/css">\n'.format(name)
        )
        r.append(
            '\n'.format(name)
        )

    if title:
        r.append("<title>{0}</title>\n".format(title))

    r.append("\n\n")
    return "".join(r)

def trans_entry(data):
    r = []
    meta = {}

    try:
        root = et.fromstring(data)
        head = root.find("Head")
        title = _get_text_nr(head.find("HWD/BASE"))
        poslist = head.findall("POS")
        if poslist:
            title += " ({0})".format(", ".join(_get_text_nr(pos) for pos in poslist))
    except:
        title = ""

    try:
        pron_gb = head.find('Audio[@resource="GB_HWD_PRON"]')
        if pron_gb is not None:
            meta["gb_pron"] = pron_gb.get("topic").split("/")[-1]
        pron_us = head.find('Audio[@resource="US_HWD_PRON"]')
        if pron_us is not None:
            meta["us_pron"] = pron_us.get("topic").split("/")[-1]
    except:
        pass

    r.append(_build_header(["entry"], title=title, meta=meta))

    r.append(et.tounicode(_trans_assets(root), pretty_print=True, method="html"))

    r.append(body2html(root))
    r.append("")
    return enc_utf8("".join(r))

def trans_thesaurus(data_set):
    r = []
    r.append(_build_header(["thesaurus"]))
    for data in data_set:
        root = et.fromstring(data)
        head = root.find("SECHEADING")
        if head is not None:
            r.append("
{0}
".format(head.text))

        for exp in root.iterfind("Exponent"):
            r.append('
')
            head = exp.find("exp-head/EXP").text
            body = body2html(exp.find("exp-body"))
            r.append('{0} {1}'.format(escape(head), body))
            r.append("
\n")

    r.append("\n")
    return enc_utf8("".join(r))

def trans_collocations(data):
    r = []
    r.append(_build_header(["collocations"]))
    root = et.fromstring(data)
    for cb in root.iterfind("ColloBox"):
        head = cb.find("HEADING")

for sec in cb.iterfind("Section"):
            head = sec.find("SECHEADING")
            if head is not None:
                r.append("
{0}
\n".format(head.text))

            for co in sec.iterfind("Collocate"):
                r.append("
")
                head = co.find("coll-head/COLLOC").text
                r.append("" + escape(head) + " ")
                r.append(body2html(co.find("coll-body")))
                r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_word_sets(data_set):
    r = []
    r.append(_build_header(["word_sets"]))
    for data in data_set:
        root = et.fromstring(data)
        name = root.find("ws-head/name").text
        number = root.find("ws-head/number").text
        r.append("
{0} ({1})
\n".format(name, number))
        r.append("
\n")
        for ref in root.iterfind("ws-body/Ref"):
            hwd = ref.find("hwd").text or ""
            pos = ref.find("pos").text or ""
            topic = ref.get("topic")
            link = "/fs/" + shorten_id(topic)
            r.append(
                """
{1} """
                """{2}"""
                """
\n""".format(link, escape(hwd), escape(pos))
            )

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_phrases(data):
    r = []
    r.append(_build_header(["phrases"]))
    root = et.fromstring(data)
    for ph in root.iterfind("phrase"):
        ref = ph.find("phrase-head/Ref")
        topic = ref.get("topic")
        bookmark = ref.get("bookmark")
        phrase = ref.text
        link = "/fs/{0}#{1}".format(shorten_id(topic), shorten_id(bookmark))
        r.append('
{0}
'.format(escape(phrase), link))
        r.append("
")
        for e in (_get_text_r(e) for e in ph.iterfind("phrase-body/exa")):
            r.append("
{0}
".format(e))

        r.append("
")

    r.append("")
    return enc_utf8("".join(r))

def trans_examples(data):
    root = et.fromstring(data)
    title = '{0} {1}'.format(
        root.find("exa-head/hwd").text, root.find("exa-head/pos").text
    )
    exas = tuple
    r = []
    r.append(_build_header(["examples"]))
    r.append("
{0}
\n".format(title))
    r.append("
\n")
    for e in (_get_text_r(e) for e in root.iterfind("exa-body/exa")):
        r.append('
{0}
\n'.format(e))

    r.append("
\n")
    r.append("\n")
    return enc_utf8("".join(r))

def trans_word_families(data):
    r = []
    r.append(_build_header(["word_families"]))
    root = et.fromstring(data)
    for g in root.iterfind("group"):
        pos = g.find("pos").text
        r.append("
{0}
".format(pos))
        r.append("
")
        for w in g.iterfind("w"):
            r.append("
")
            ref = w.find("Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    '{1}'.format(link, escape(ref.text))
                )
            else:
                r.append(_get_text_r(w))

            ref = w.find("opp/Ref")
            if ref is not None:
                link = "/fs/" + shorten_id(ref.get("topic"))
                r.append(
                    ' ≠ {1}'.format(
                        link, escape(ref.text)
                    )
                )

            r.append("
\n")
        r.append("
\n")

    r.append("\n")

    return enc_utf8("".join(r))

def trans_etymologies(data):
    r = []
    r.append(_build_header(["etymologies"]))
    root = et.fromstring(data)
    r.append(body2html(root))
    r.append("\n")
    return enc_utf8("".join(r))

def _trans_activator_concept(data, sid):
    r = []
    root = et.fromstring(data)
    cid = root.get("id")

    def section(e):
        nr = e.find("SECNR")
        if nr is not None:
            r.append('
{0}.
\n'.format(_get_text_nr(nr)))

        cls = "section sel" if (e.get("id") == sid) else "section"
        r.append('
\n'.format(cls))
        r.append(
            '{text}'.format(
                cid=cid, sid=e.get("id"), text=_get_text_nr(e)
            )
        )
        r.append("
\n")

    def ref(e):
        r.append(
            '
{text}
'.format(
                cid=e.get("topic"),
                sid=e.get("selection"),
                text=escape(e.text.replace("/", " / ")),
            )
        )

    def reference(e):
        reftype = e.find("REFTYPE")
        if reftype is not None:
            r.append('{0}:'.format(escape(reftype.text)))

        r.append("
")
        for cr in e.iterfind("Crossref/Ref"):
            ref(cr)

        r.append("
")

    title = root.find("HWD").text
    r.append("
{0}
".format(escape(title.replace("/", " / "))))
    for e in root:
        if e.tag == "SUBHWD":
            r.append("
{0}
".format(escape(e.text.replace("/", " / "))))
        elif e.tag == "Section":
            section(e)

    r.append("
Related Words
")
    r.append("
")
    for e in root.iterfind("References/Reference"):
        r.append("
")
        reference(e)
        r.append("
")

    r.append("
")
    return "".join(r), title

def _trans_activator_section(data):
    r = []

    def exponent(e):
        r.append(body2html(e))

    root = et.fromstring(data)
    plain_title = title = _get_text_nr(root.find("SECDEF"))
    secnr = root.find("SECDEF/SECNR")
    if secnr is not None:
        nr = secnr.text.strip()
        plain_title = "({0}) - {1}".format(nr, title)
        title = "{0}. {1}".format(nr, title)

    r.append("
{0}
".format(title))
    for e in root.iterfind("Exponent"):
        exponent(e)

    return "".join(r), plain_title

def trans_activator(data_c, data_s, sid):
    r = []
    (concept_body, concept_title) = _trans_activator_concept(data_c, sid)
    (section_body, section_title) = _trans_activator_section(data_s)
    title = concept_title + section_title
    r.append(_build_header(["activator"], title=title))
    # section
    r.append('
\n')
    r.append('
\n')
    r.append(section_body)
    r.append("
\n")
    r.append("
\n")
    # concept
    r.append('
\n')
    r.append(concept_body)
    r.append("
\n")
    # end
    r.append("")
    return enc_utf8("".join(r))