import http.client
import os
import urllib.parse

import tornado.web
import prometheus_client

from app.master.slave import SlaveRegistry
from app.util import analytics
from app.util import log
from app.util.conf.configuration import Configuration
from app.util.decorators import authenticated
from app.util.exceptions import ItemNotFoundError
from app.util.url_builder import UrlBuilder
from app.web_framework.cluster_application import ClusterApplication
from app.web_framework.cluster_base_handler import ClusterBaseAPIHandler, ClusterBaseHandler
from app.web_framework.route_node import RouteNode

#   Handler classes are not designed to have __init__ overridden.

class ClusterMasterApplication(ClusterApplication):

    def __init__(self, cluster_master):
        """
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        default_params = {
            'cluster_master': cluster_master,
        }
        # The routes are described using a tree structure.  This is a better representation of a path than a flat list
        #  of strings and allows us to inspect children/parents of a node to generate 'child routes'
        api_v1 = [
            RouteNode(r'v1', _APIVersionOneHandler).add_children([
                RouteNode(r'metrics', _MetricsHandler),
                RouteNode(r'version', _VersionHandler),
                RouteNode(r'build', _BuildsHandler, 'builds').add_children([
                    RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                        RouteNode(r'result', _BuildResultRedirectHandler),
                        RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                        RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                        RouteNode(r'subjob', _SubjobsHandler, 'subjobs').add_children([
                            RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                                RouteNode(r'atom', _AtomsHandler, 'atoms').add_children([
                                    RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                        RouteNode(r'console', _AtomConsoleHandler),
                                    ]),
                                ]),
                                RouteNode(r'result', _SubjobResultHandler),
                            ]),
                        ]),
                    ]),
                ]),
                RouteNode(r'queue', _QueueHandler),
                RouteNode(r'slave', _SlavesHandler, 'slaves').add_children([
                    RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                        RouteNode(r'shutdown', _SlaveShutdownHandler, 'shutdown'),
                        RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                    ]),
                    RouteNode(r'shutdown', _SlavesShutdownHandler, 'shutdown'),
                ]),
                RouteNode(r'eventlog', _EventlogHandler)])]

        api_v2 = [
            RouteNode(r'metrics', _MetricsHandler),
            RouteNode(r'version', _VersionHandler),
            RouteNode(r'builds', _V2BuildsHandler).add_children([
                RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                    RouteNode(r'result', _BuildResultRedirectHandler),
                    RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                    RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                    RouteNode(r'subjobs', _V2SubjobsHandler,).add_children([
                        RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                            RouteNode(r'atoms', _V2AtomsHandler).add_children([
                                RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                    RouteNode(r'console', _AtomConsoleHandler),
                                ]),
                            ]),
                            RouteNode(r'result', _SubjobResultHandler),
                        ]),
                    ]),
                ]),
            ]),
            RouteNode(r'queue', _QueueHandler),
            RouteNode(r'slaves', _SlavesHandler).add_children([
                RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                    RouteNode(r'shutdown', _SlaveShutdownHandler),
                    RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                ]),
                RouteNode(r'shutdown', _SlavesShutdownHandler),
            ]),
            RouteNode(r'eventlog', _EventlogHandler)]

        root = RouteNode(r'/', _RootHandler)
        root.add_children(a

        root.add_children(api_v2, version=2)

        handlers = self.get_all_handlers(root, default_params)
        super().__init__(handlers)

class _ClusterMasterBaseAPIHandler(ClusterBaseAPIHandler):
    def initialize(self, route_node=None, cluster_master=None):
        """
        :type route_node: RouteNode
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        self._logger = log.get_logger(__name__)
        self._cluster_master = cluster_master
        super().initialize(route_node)

class _RootHandler(_ClusterMasterBaseAPIHandler):
    pass

class _APIVersionOneHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'master': self._cluster_master.api_representation(),
        }
        self.write(response)

class _VersionHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'version': Configuration['version'],
            'api_version': self.api_version,
        }
        self.write(response)

class _MetricsHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        self.write_text(prometheus_client.exposition.generate_latest(prometheus_client.core.REGISTRY))

class _QueueHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'queue': [build.api_representation() for build in self._cluster_master.active_builds()]
        }
        self.write(response)

class _SubjobsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id):
        build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs()]
        }
        self.write(response)

class _V2SubjobsHandler(_SubjobsHandler):
    def get(self, build_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs(offset, limit)]
        }
        self.write(response)

class _SubjobHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'subjob': subjob.api_representation()
        }
        self.write(response)

class _SubjobResultHandler(_ClusterMasterBaseAPIHandler):
    def post(self, build_id, subjob_id):
        slave_url = self.decoded_body.get('slave')
        slave = SlaveRegistry.singleton().get_slave(slave_url=slave_url)
        file_payload = self.request.files.get('file')
        if not file_payload:
            raise RuntimeError('Result file not provided')

        slave_executor_id = self.decoded_body.get('metric_data', {}).get('executor_id')
        analytics.record_event(analytics.MASTER_RECEIVED_RESULT, executor_id=slave_executor_id, build_id=int(build_id),
                               subjob_id=int(subjob_id), slave_id=slave.id)

        self._cluster_master.handle_result_reported_from_slave(
            slave_url, int(build_id), int(subjob_id), file_payload[0])
        self._write_status()

    def get(self, build_id, subjob_id):
        # TODO: return the subjob's result archive here?
        self.write({'status': 'not implemented'})

class _AtomsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.atoms()],
        }
        self.write(response)

class _V2AtomsHandler(_AtomsHandler):
    def get(self, build_id, subjob_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.get_atoms(offset, limit)],
        }
        self.write(response)

class _AtomHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        atoms = subjob.atoms
        response = {
            'atom': atoms[int(atom_id)].api_representation(),
        }
        self.write(response)

class _AtomConsoleHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        """
        :type build_id: int
        :type subjob_id: int
        :type atom_id: int
        """
        max_lines = int(self.get_query_argument('max_lines', 50))
        offset_line = self.get_query_argument('offset_line', None)

        if offset_line is not None:
            offset_line = int(offset_line)

        try:
            response = self._cluster_master.get_console_output(
                build_id,
                subjob_id,
                atom_id,
                Configuration['results_directory'],
                max_lines,
                offset_line
            )
            self.write(response)
            return
        except ItemNotFoundError as e:
            # If the master doesn't have the atom's console output, it's possible it's currently being worked on,
            # in which case the slave that is working on it may be able to provide the in-progress console output.
            build = self._cluster_master.get_build(int(build_id))
            subjob = build.subjob(int(subjob_id))
            slave = subjob.slave

            if slave is None:
                raise e

            api_url_builder = UrlBuilder(slave.url)
            slave_console_url = api_url_builder.url('build', build_id, 'subjob', subjob_id, 'atom', atom_id, 'console')
            query = {'max_lines': max_lines}

            if offset_line is not None:
                query['offset_line'] = offset_line

            query_string = urllib.parse.urlencode(query)
            self.redirect('{}?{}'.format(slave_console_url, query_string))

class _BuildsHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        build_params = self.decoded_body
        success, response = self._cluster_master.handle_request_for_new_build(build_params)
        status_code = http.client.ACCEPTED if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self):
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds()]
        }
        self.write(response)

class _V2BuildsHandler(_BuildsHandler):
    def get(self):
        offset, limit = self.get_pagination_params()
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds(offset, limit)]
        }
        self.write(response)

class _BuildHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def put(self, build_id):
        update_params = self.decoded_body
        success, response = self._cluster_master.handle_request_to_update_build(build_id, update_params)
        status_code = http.client.OK if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self, build_id):
        response = {
            'build': self._cluster_master.get_build(int(build_id)).api_representation(),
        }
        self.write(response)

class _BuildResultRedirectHandler(_ClusterMasterBaseAPIHandler):
    """
    Redirect to the actual build results file download URL.
    """
    def get(self, build_id):
        self.redirect('/v1/build/{}/artifacts.tar.gz'.format(build_id))

class _BuildResultHandler(ClusterBaseHandler, tornado.web.StaticFileHandler):
    """
    Download an artifact for the specified build. Note this class inherits from ClusterBaseHandler and
    StaticFileHandler, so the semantics of this handler are a bit different than the other handlers in this file that
    inherit from _ClusterMasterBaseHandler.

    From the Tornado docs: "for heavy traffic it will be more efficient to use a dedicated static file server".
    """
    def initialize(self, route_node=None, cluster_master=None):
        """
        :param route_node: This is not used, it is only a param so we can pass route_node to all handlers without error.
        In other routes, route_node is used to find child routes but filehandler routes will never show child routes.
        :type route_node: RouteNode | None
        :type cluster_master: app.master.cluster_master.ClusterMaster | None
        """
        self._cluster_master = cluster_master
        super().initialize(path=None)  # we will not set the root path until the get() method is called

    def get(self, build_id):
        artifact_file_path = self.get_result_file_download_path(int(build_id))
        self.root, artifact_filename = os.path.split(artifact_file_path)
        self.set_header('Content-Type', 'application/octet-stream')  # this should be downloaded as a binary file
        return super().get(path=artifact_filename)

    def get_result_file_download_path(self, build_id: int):
        raise NotImplementedError

class _BuildTarResultHandler(_BuildResultHandler):
    """Handler for the tar archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.tar.gz for the specified build."""
        return self._cluster_master.get_path_for_build_results_archive(build_id, is_tar_request=True)

class _BuildZipResultHandler(_BuildResultHandler):
    """Handler for the zip archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.zip for the specified build."""
        return self._cluster_master.get_path_for_build_results_archive(build_id)

class _SlavesHandler(_ClusterMasterBaseAPIHandler):
    def post(self):
        slave_url = self.decoded_body.get('slave')
        num_executors = int(self.decoded_body.get('num_executors'))
        session_id = self.decoded_body.get('session_id')
        response = self._cluster_master.connect_slave(slave_url, num_executors, session_id)
        self._write_status(response, status_code=201)

    def get(self):

        response = {
            'slaves': [slave.api_representation() for slave in SlaveRegistry.singleton().get_all_slaves_by_id().values()]
        }
        self.write(response)

class _SlaveHandler(_ClusterMasterBaseAPIHandler):
    def get(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        response = {
            'slave': slave.api_representation()
        }
        self.write(response)

    @authenticated
    def put(self, slave_id):
        new_slave_state = self.decoded_body.get('slave', {}).get('state')
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        self._cluster_master.handle_slave_state_update(slave, new_slave_state)
        self._cluster_master.update_slave_last_heartbeat_time(slave)

        self._write_status({
            'slave': slave.api_representation()
        })

class _EventlogHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        # all arguments are optional, so default to None
        since_timestamp = self.get_query_argument('since_timestamp', None)
        since_id = self.get_query_argument('since_id', None)
        self.write({
            'events': analytics.get_events(since_timestamp, since_id),
        })

class _SlaveShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slaves_to_shutdown = [int(slave_id)]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        shutdown_all = self.decoded_body.get('shutdown_all')
        if shutdown_all:
            slaves_to_shutdown = SlaveRegistry.singleton().get_all_slaves_by_id().keys()
        else:
            slaves_to_shutdown = [int(slave_id) for slave_id in self.decoded_body.get('slaves')]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesHeartbeatHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))

        # If the slave has been marked dead, but still sends heartbeat, the master does not update the last
        # heartbeat time and the method returns false. Additionally, master responds to the slave with slave
        # status. The slave will treat a is_alive=false response as a heartbeat failure, and die.
        # The reason master returns the status to the slave instead of simply marking the slave as alive is
        # because the master or slave do not maintain an explicit state about when and why the slave was marked
        # dead. It is a lot cleaner for the heartbeat functionality to indicate an heartbeat failure and let the
        # slave make a decision based on that.

        is_alive = self._cluster_master.update_slave_last_heartbeat_time(slave)
        self.write({'is_alive': is_alive})


import http.client
import os
import urllib.parse

import tornado.web
import prometheus_client

from app.master.slave import SlaveRegistry
from app.util import analytics
from app.util import log
from app.util.conf.configuration import Configuration
from app.util.decorators import authenticated
from app.util.exceptions import ItemNotFoundError
from app.util.url_builder import UrlBuilder
from app.web_framework.cluster_application import ClusterApplication
from app.web_framework.cluster_base_handler import ClusterBaseAPIHandler, ClusterBaseHandler
from app.web_framework.route_node import RouteNode

#   Handler classes are not designed to have __init__ overridden.

class ClusterMasterApplication(ClusterApplication):

    def __init__(self, cluster_master):
        """
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        default_params = {
            'cluster_master': cluster_master,
        }
        # The routes are described using a tree structure.  This is a better representation of a path than a flat list
        #  of strings and allows us to inspect children/parents of a node to generate 'child routes'
        api_v1 = [
            RouteNode(r'v1', _APIVersionOneHandler).add_children([
                RouteNode(r'metrics', _MetricsHandler),
                RouteNode(r'version', _VersionHandler),
                RouteNode(r'build', _BuildsHandler, 'builds').add_children([
                    RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                        RouteNode(r'result', _BuildResultRedirectHandler),
                        RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                        RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                        RouteNode(r'subjob', _SubjobsHandler, 'subjobs').add_children([
                            RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                                RouteNode(r'atom', _AtomsHandler, 'atoms').add_children([
                                    RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                        RouteNode(r'console', _AtomConsoleHandler),
                                    ]),
                                ]),
                                RouteNode(r'result', _SubjobResultHandler),
                            ]),
                        ]),
                    ]),
                ]),
                RouteNode(r'queue', _QueueHandler),
                RouteNode(r'slave', _SlavesHandler, 'slaves').add_children([
                    RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                        RouteNode(r'shutdown', _SlaveShutdownHandler, 'shutdown'),
                        RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                    ]),
                    RouteNode(r'shutdown', _SlavesShutdownHandler, 'shutdown'),
                ]),
                RouteNode(r'eventlog', _EventlogHandler)])]

        api_v2 = [
            RouteNode(r'metrics', _MetricsHandler),
            RouteNode(r'version', _VersionHandler),
            RouteNode(r'builds', _V2BuildsHandler).add_children([
                RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                    RouteNode(r'result', _BuildResultRedirectHandler),
                    RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                    RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                    RouteNode(r'subjobs', _V2SubjobsHandler,).add_children([
                        RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                            RouteNode(r'atoms', _V2AtomsHandler).add_children([
                                RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                    RouteNode(r'console', _AtomConsoleHandler),
                                ]),
                            ]),
                            RouteNode(r'result', _SubjobResultHandler),
                        ]),
                    ]),
                ]),
            ]),
            RouteNode(r'queue', _QueueHandler),
            RouteNode(r'slaves', _SlavesHandler).add_children([
                RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                    RouteNode(r'shutdown', _SlaveShutdownHandler),
                    RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                ]),
                RouteNode(r'shutdown', _SlavesShutdownHandler),
            ]),
            RouteNode(r'eventlog', _EventlogHandler)]

        root = RouteNode(r'/', _RootHandler)
        root.add_children(api_v1, version=1)
        root.add_children(api_v2, version=2)

        handlers = self.get_all_handlers(root, default_params)
        super().__init__(handlers)

class _ClusterMasterBaseAPIHandler(ClusterBaseAPIHandler):
    def initialize(self, route_node=None, cluster_master=None):
        """
        :type route_node: RouteNode
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        self._logger = log.get_logger(__name__)
        self._cluster_master = cluster_master
        super().initialize(route_node)

class _RootHandler(_ClusterMasterBaseAPIHandler):
    pass

class _APIVersionOneHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'master': self._cluster_master.api_representation(),
        }
        self.write(response)

class _VersionHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'version': Configuration['version'],
            'api_version': self.api_version,
        }
        self.write(response)

class _MetricsHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        self.write_text(prometheus_client.exposition.generate_latest(prometheus_client.core.REGISTRY))

class _QueueHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'queue': [build.api_representation() for build in self._cluster_master.active_builds()]
        }
        self.write(response)

class _SubjobsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id):
        build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs()]
        }
        self.write(response)

class _V2SubjobsHandler(_SubjobsHandler):
    def get(self, build_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs(offset, limit)]
        }
        self.write(response)

class _SubjobHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'subjob': subjob.api_representation()
        }
        self.write(response)

class _SubjobResultHandler(_ClusterMasterBaseAPIHandler):
    def post(self, build_id, subjob_id):
        slave_url = self.decoded_body.get('slave')
        slave = SlaveRegistry.singleton().get_slave(slave_url=slave_url)
        file_payload = self.request.files.get('file')
        if not file_payload:
            raise RuntimeError('Result file not provided')

        slave_executor_id = self.decoded_body.get('metric_data', {}).get('executor_id')
        analytics.record_event(analytics.MASTER_RECEIVED_RESULT, executor_id=slave_executor_id, build_id=int(build_id),
                               subjob_id=int(subjob_id), slave_id=slave.id)

        self._cluster_master.handle_result_reported_from_slave(
            slave_url, int(build_id), int(subjob_id), file_payload[0])
        self._write_status()

    def get(self, build_id, subjob_id):
        # TODO: return the subjob's result archive here?
        self.write({'status': 'not implemented'})

class _AtomsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.atoms()],
        }
        self.write(response)

class _V2AtomsHandler(_AtomsHandler):
    def get(self, build_id, subjob_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.get_atoms(offset, limit)],
        }
        self.write(response)

class _AtomHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        atoms = subjob.atoms
        response = {
            'atom': atoms[int(atom_id)].api_representation(),
        }
        self.write(response)

class _AtomConsoleHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        """
        :type build_id: int
        :type subjob_id: int
        :type atom_id: int
        """
        max_lines = int(self.get_query_argument('max_lines', 50))
        offset_line = self.get_query_argument('offset_line', None)

        if offset_line is not None:
            offset_line = int(offset_line)

        try:
            response = self._cluster_master.get_console_output(
                build_id,
                subjob_id,
                atom_id,
                Configuration['results_directory'],
                max_lines,
                offset_line
            )
            self.write(response)
            return
        except ItemNotFoundError as e:
            # If the master doesn't have the atom's console output, it's possible it's currently being worked on,
            # in which case the slave that is working on it may be able to provide the in-progress console output.
            build = self._cluster_master.get_build(int(build_id))
            subjob = build.subjob(int(subjob_id))
            slave = subjob.slave

            if slave is None:
                raise e

            api_url_builder = UrlBuilder(slave.url)
            slave_console_url = api_url_builder.url('build', build_id, 'subjob', subjob_id, 'atom', atom_id, 'console')
            query = {'max_lines': max_lines}

            if offset_line is not None:
                query['offset_line'] = offset_line

            query_string = urllib.parse.urlencode(query)
            self.redirect('{}?{}'.format(slave_console_url, query_string))

class _BuildsHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        build_params = self.decoded_body
        success, response = self._cluster_master.handle_request_for_new_build(build_params)
        status_code = http.client.ACCEPTED if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self):
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds()]
        }
        self.write(response)

class _V2BuildsHandler(_BuildsHandler):
    def get(self):
        offset, limit = self.get_pagination_params()
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds(offset, limit)]
        }
        self.write(response)

class _BuildHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def put(self, build_id):
        update_params = self.decoded_body
        success, response = self._cluster_master.handle_request_to_update_build(build_id, update_params)
        status_code = http.client.OK if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self, build_id):
        response = {
            'build': self._cluster_master.get_build(int(build_id)).api_representation(),
        }
        self.write(response)

class _BuildResultRedirectHandler(_ClusterMasterBaseAPIHandler):
    """
    Redirect to the actual build results file download URL.
    """
    def get(self, build_id):
        self.redirect('/v1/build/{}/artifacts.tar.gz'.format(build_id))

class _BuildResultHandler(ClusterBaseHandler, tornado.web.StaticFileHandler):
    """
    Download an artifact for the specified build. Note this class inherits from ClusterBaseHandler and
    StaticFileHandler, so the semantics of this handler are a bit different than the other handlers in this file that
    inherit from _ClusterMasterBaseHandler.

    From the Tornado docs: "for heavy traffic it will be more efficient to use a dedicated static file server".
    """
    def initialize(self, route_node=None, cluster_master=None):
        """
        :param route_node: This is not used, it is only a param so we can pass route_node to all handlers without error.
        In other routes, route_node is used to find child routes but filehandler routes will never show child routes.
        :type route_node: RouteNode | None
        :type cluster_master: app.master.cluster_master.ClusterMaster | None
        """
        self._cluster_master = cluster_master
        super().initialize(path=None)  # we will not set the root path until the get() method is called

    def get(self, build_id):
        artifact_file_path = self.get_result_file_download_path(int(build_id))
        self.root, artifact_filename = os.path.split(artifact_file_path)
        self.set_header('Content-Type', 'application/octet-stream')  # this should be downloaded as a binary file
        return super().get(path=artifact_filename)

    def get_result_file_download_path(self, build_id: int):
        raise NotImplementedError

class _BuildTarResultHandler(_BuildResultHandler):
    """Handler for the tar archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.tar.gz for the specified build."""

class _BuildZipResultHandler(_BuildResultHandler):
    """Handler for the zip archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.zip for the specified build."""
        return self._cluster_master.get_path_for_build_results_archive(build_id)

class _SlavesHandler(_ClusterMasterBaseAPIHandler):
    def post(self):
        slave_url = self.decoded_body.get('slave')
        num_executors = int(self.decoded_body.get('num_executors'))
        session_id = self.decoded_body.get('session_id')
        response = self._cluster_master.connect_slave(slave_url, num_executors, session_id)
        self._write_status(response, status_code=201)

    def get(self):

        response = {
            'slaves': [slave.api_representation() for slave in SlaveRegistry.singleton().get_all_slaves_by_id().values()]
        }
        self.write(response)

class _SlaveHandler(_ClusterMasterBaseAPIHandler):
    def get(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        response = {
            'slave': slave.api_representation()
        }
        self.write(response)

    @authenticated
    def put(self, slave_id):
        new_slave_state = self.decoded_body.get('slave', {}).get('state')
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        self._cluster_master.handle_slave_state_update(slave, new_slave_state)
        self._cluster_master.update_slave_last_heartbeat_time(slave)

        self._write_status({
            'slave': slave.api_representation()
        })

class _EventlogHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        # all arguments are optional, so default to None
        since_timestamp = self.get_query_argument('since_timestamp', None)
        since_id = self.get_query_argument('since_id', None)
        self.write({
            'events': analytics.get_events(since_timestamp, since_id),
        })

class _SlaveShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slaves_to_shutdown = [int(slave_id)]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        shutdown_all = self.decoded_body.get('shutdown_all')
        if shutdown_all:
            slaves_to_shutdown = SlaveRegistry.singleton().get_all_slaves_by_id().keys()
        else:
            slaves_to_shutdown = [int(slave_id) for slave_id in self.decoded_body.get('slaves')]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesHeartbeatHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))

        # If the slave has been marked dead, but still sends heartbeat, the master does not update the last
        # heartbeat time and the method returns false. Additionally, master responds to the slave with slave
        # status. The slave will treat a is_alive=false response as a heartbeat failure, and die.
        # The reason master returns the status to the slave instead of simply marking the slave as alive is
        # because the master or slave do not maintain an explicit state about when and why the slave was marked
        # dead. It is a lot cleaner for the heartbeat functionality to indicate an heartbeat failure and let the
        # slave make a decision based on that.

        is_alive = self._cluster_master.update_slave_last_heartbeat_time(slave)
        self.write({'is_alive': is_alive})


if out_folder is None:
        raise ValueError("None type")
    out_path = os.path.join(out_folder, name)
    np.savez(out_path, **kwargs)

import os
import numpy as np

def save_np_arrays(name, out_folder=None, **kwargs):


import os
import numpy as np

def save_np_arrays(name, out_folder=None, **kwargs):
    if out_folder is None:
        raise ValueError("None type")
    out_path = os.path.join(out_folder, name)
    np.sa


dna_letters='AaTtUuGgCcNn'
    ndna=0
    nU=0
    nT=0
    for l in s:
        if l in dna_letters:
            ndna += 1
        if l=='U' or l=='u':
            nU += 1
        elif l=='T' or l=='t':
            nT += 1
    ratio=ndna/float(len(s))
    if ratio>0.85:
        if nT>nU:
            return DNA_SEQTYPE
        else:
            return RNA_SEQTYPE
    else:
        return PROTEIN_SEQTYPE

DNA_SEQTYPE=0
RNA_SEQTYPE=1
PROTEIN_SEQTYPE=2

def guess_seqtype(s):

seq_id_counter=0

def new_seq_id():
    global seq_id_counter
    seq_id_counter += 1
    return str(seq_id_counter-1)

def write_fasta(ofile, s, chunk=60, id=None, reformatter=None):
    "Trivial FASTA output"
    if id is None:
        try:
            id = str(s.id)
        except AttributeError:
            id = new_seq_id()

    ofile.write('>' + id + '\n')
    seq = str(s)
    if reformatter is not None: # APPLY THE DESIRED REFORMATTING
        seq = reformatter(seq)
    end = len(seq)
    pos = 0
    while 1:
        ofile.write(seq[pos:pos+chunk] + '\n')
        pos += chunk
        if pos >= end:
            break
    return id # IN CASE CALLER WANTS TEMP ID WE MAY HAVE ASSIGNED

def read_fasta(ifile):
    "iterate over id,title,seq from stream ifile"
    id = None
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and len(seq) > 0:
                yield id, title, seq
                isEmpty = False
            id = line[1:].split()[0]
            title = line[len(id)+2:]
            seq = ''
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
    if id is not None and len(seq) > 0:
        yield id, title, seq
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

def read_fasta_one_line(ifile): # @CTB deprecated; remove
    "read a single sequence line, return id,title,seq"
    id = None
    seq = ''
    while True:
        line = ifile.readline(1024) # READ AT MOST 1KB
        if line == '': # EOF
            break
        elif '>' == line[0]:
            id = line[1:].split()[0]
            title = line[len(id)+2:]
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
            if len(seq) > 0:
                return id, title, seq
    raise IOError('no readable sequence in FASTA file!')

def read_fasta_lengths(ifile):
    "Generate sequence ID,length from stream ifile"
    id = None
    seqLength = 0
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and seqLength > 0:
                yield id, seqLength
                isEmpty = False
            id = line[1:].split()[0]
            seqLength = 0
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seqLength += len(word)
    if id is not None and seqLength > 0:
        yield id, seqLength
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

class AATranslation(object):
    'customizable translation class'
    geneticCode = dict(TTY='F', TTR='L', TCN='S', TAY='Y', TGY='C', TGG='W',
                       CTN='L', CCN='P', CAY='H', CAR='Q', CGN='R',
                       ATY='I', ATA='I', ATG='M', ACN='T', AAY='N', AAR='K',
                       AGY='S', AGR='R',
                       GTN='V', GCN='A', GAY='D', GAR='E', GGN='G',
                       TAR='*', TGA='*')

    def __init__(self):
        'initialize our translation dictionary by applying N,Y,R codes'
        geneticCode = self.geneticCode.copy()
        for codon, aa in self.geneticCode.items():
            if codon[2] == 'N':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'G'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'Y':
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'R':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'G'] = aa
        self.geneticCode = geneticCode

    def __call__(self, s):
        'translate nucleotide string s to amino acid string'
        s = s.upper()
        s = s.replace('U', 'T')
        l = []
        for i in range(0, len(s), 3):
            try:
                l.append(self.geneticCode[s[i:i+3]])
            except KeyError:
                l.append('X') # uninterpretable
        return ''.join(l)

translate_orf = AATranslation() # default translation function


DNA_SEQTYPE=0
RNA_SEQTYPE=1
PROTEIN_SEQTYPE=2

def guess_seqtype(s):
    dna_letters='AaTtUuGgCcNn'
    ndna=0
    nU=0
    nT=0
    for l in s:
        if l in dna_letters:
            ndna += 1
        if l=='U' or l=='u':
            nU += 1
        elif l=='T' or l=='t':
            nT += 1
    ratio=ndna/float(len(s))
    if ratio>0.85:
        if nT>nU:
            return DNA_SEQTYPE
        else:
            return RNA_SEQTYPE
    else:
        return PROTEIN_SEQTYPE

seq_id_counter=0

def new_seq_id():
    global seq_id_counter
    seq_id_counter += 1
    return str(seq_id_counter-1)

def write_fasta(ofile, s, chunk=60, id=None, reformatter=None):
    "Trivial FASTA output"
    if id is None:
        try:
            id = str(s.id)
        except AttributeError:
            id = new_seq_id()

    ofile.write('>' + id + '\n')
    seq = str(s)
    if reformatter is not None: # APPLY THE DESIRED REFORMATTING
        seq = reformatter(seq)
    end = len(seq)
    pos = 0
    while 1:
        ofile.write(seq[pos:pos+chunk] + '\n')
        pos += chunk
        if pos >= end:
            break
    return id # IN CASE CALLER WANTS TEMP ID WE MAY HAVE ASSIGNED

def read_fasta(ifile):
    "iterate over id,title,seq from stream ifile"
    id = None
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and len(seq) > 0:
                yield id, title, seq
                isEmpty = False
            id = line[1:].split()[0]
            title = line[len(id)+2:]
            seq = ''
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
    if id is not None and len(seq) > 0:
        yield id, title, seq
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

def read_fasta_one_line(ifile): # @CTB deprecated; remove
    "read a single sequence line, return id,title,seq"
    id = None
    seq = ''
    while True:
        line = ifile.readline(1024) # READ AT MOST 1KB
        if line == '': # EOF
            break
        elif '>' == line[0]:
            id = line[1:].split()[0]
            title = line[len(id)+2:]
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
            if len(seq) > 0:
                return id, title, seq
    raise IOError('no readable sequence in FASTA file!')

def read_fasta_lengths(ifile):
    "Generate sequence ID,length from stream ifile"
    id =

    seqLength = 0
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and seqLength > 0:
                yield id, seqLength
                isEmpty = False
            id = line[1:].split()[0]
            seqLength = 0
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seqLength += len(word)
    if id is not None and seqLength > 0:
        yield id, seqLength
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

class AATranslation(object):
    'customizable translation class'
    geneticCode = dict(TTY='F', TTR='L', TCN='S', TAY='Y', TGY='C', TGG='W',
                       CTN='L', CCN='P', CAY='H', CAR='Q', CGN='R',
                       ATY='I', ATA='I', ATG='M', ACN='T', AAY='N', AAR='K',
                       AGY='S', AGR='R',
                       GTN='V', GCN='A', GAY='D', GAR='E', GGN='G',
                       TAR='*', TGA='*')

    def __init__(self):
        'initialize our translation dictionary by applying N,Y,R codes'
        geneticCode = self.geneticCode.copy()
        for codon, aa in self.geneticCode.items():
            if codon[2] == 'N':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'G'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'Y':
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'R':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'G'] = aa
        self.geneticCode = geneticCode

    def __call__(self, s):
        'translate nucleotide string s to amino acid string'
        s = s.upper()
        s = s.replace('U', 'T')
        l = []
        for i in range(0, len(s), 3):
            try:
                l.append(self.geneticCode[s[i:i+3]])
            except KeyError:
                l.append('X') # uninterpretable
        return ''.join(l)

translate_orf = AATranslation() # default translation function


        s = s.upper()
        s = s.replace('U', 'T')
        l = []
        for i in range(0, len(s), 3):
            try:
                l.append(self.geneticCode[s[i:i+3]])
            except KeyError:
                l.append('X') # uninterpretable
        return ''.join(l)

DNA_SEQTYPE=0
RNA_SEQTYPE=1
PROTEIN_SEQTYPE=2

def guess_seqtype(s):
    dna_letters='AaTtUuGgCcNn'
    ndna=0
    nU=0
    nT=0
    for l in s:
        if l in dna_letters:
            ndna += 1
        if l=='U' or l=='u':
            nU += 1
        elif l=='T' or l=='t':
            nT += 1
    ratio=ndna/float(len(s))
    if ratio>0.85:
        if nT>nU:
            return DNA_SEQTYPE
        else:
            return RNA_SEQTYPE
    else:
        return PROTEIN_SEQTYPE

seq_id_counter=0

def new_seq_id():
    global seq_id_counter
    seq_id_counter += 1
    return str(seq_id_counter-1)

def write_fasta(ofile, s, chunk=60, id=None, reformatter=None):
    "Trivial FASTA output"
    if id is None:
        try:
            id = str(s.id)
        except AttributeError:
            id = new_seq_id()

    ofile.write('>' + id + '\n')
    seq = str(s)
    if reformatter is not None: # APPLY THE DESIRED REFORMATTING
        seq = reformatter(seq)
    end = len(seq)
    pos = 0
    while 1:
        ofile.write(seq[pos:pos+chunk] + '\n')
        pos += chunk
        if pos >= end:
            break
    return id # IN CASE CALLER WANTS TEMP ID WE MAY HAVE ASSIGNED

def read_fasta(ifile):
    "iterate over id,title,seq from stream ifile"
    id = None
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and len(seq) > 0:
                yield id, title, seq
                isEmpty = False
            id = line[1:].split()[0]
            title = line[len(id)+2:]
            seq = ''
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
    if id is not None and len(seq) > 0:
        yield id, title, seq
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

def read_fasta_one_line(ifile): # @CTB deprecated; remove
    "read a single sequence line, return id,title,seq"
    id = None
    seq = ''
    while True:
        line = ifile.readline(1024) # READ AT MOST 1KB
        if line == '': # EOF
            break
        elif '>' == line[0]:
            id = line[1:].split()[0]
            title = line[len(id)+2:]
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
            if len(seq) > 0:
                return id, title, seq
    raise IOError('no readable sequence in FASTA file!')

def read_fasta_lengths(ifile):
    "Generate sequence ID,length from stream ifile"
    id = None
    seqLength = 0
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and seqLength > 0:
                yield id, seqLength
                isEmpty = False
            id = line[1:].split()[0]
            seqLength = 0
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seqLength += len(word)
    if id is not None and seqLength > 0:
        yield id, seqLength
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

class AATranslation(object):
    'customizable translation class'
    geneticCode = dict(TTY='F', TTR='L', TCN='S', TAY='Y', TGY='C', TGG='W',
                       CTN='L', CCN='P', CAY='H', CAR='Q', CGN='R',
                       ATY='I', ATA='I', ATG='M', ACN='T', AAY='N', AAR='K',
                       AGY='S', AGR='R',
                       GTN='V', GCN='A', GAY='D', GAR='E', GGN='G',
                       TAR='*', TGA='*')

    def __init__(self):
        'initialize our translation dictionary by applying N,Y,R codes'
        geneticCode = self.geneticCode.copy()
        for codon, aa in self.geneticCode.items():
            if codon[2] == 'N':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'G'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'Y':
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'R':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'G'] = aa
        self.geneticCode = geneticCode

    def __call__(self, s):
        'translate nucleotide string s to amino acid string'

translate_orf = AATranslation() # default translation function


while 1:
        ofile.write(seq[pos:pos+chunk] + '\n')
        pos += chunk
        if pos >= end:
            break

DNA_SEQTYPE=0
RNA_SEQTYPE=1
PROTEIN_SEQTYPE=2

def guess_seqtype(s):
    dna_letters='AaTtUuGgCcNn'
    ndna=0
    nU=0
    nT=0
    for l in s:
        if l in dna_letters:
            ndna += 1
        if l=='U' or l=='u':
            nU += 1
        elif l=='T' or l=='t':
            nT += 1
    ratio=ndna/float(len(s))
    if ratio>0.85:
        if nT>nU:
            return DNA_SEQTYPE
        else:
            return RNA_SEQTYPE
    else:
        return PROTEIN_SEQTYPE

seq_id_counter=0

def new_seq_id():
    global seq_id_counter
    seq_id_counter += 1
    return str(seq_id_counter-1)

def write_fasta(ofile, s, chunk=60, id=None, reformatter=None):
    "Trivial FASTA output"
    if id is None:
        try:
            id = str(s.id)
        except AttributeError:
            id = new_seq_id()

    ofile.write('>' + id + '\n')
    seq = str(s)
    if reformatter is not None: # APPLY THE DESIRED REFORMATTING
        seq = reformatter(seq)
    end = len(seq)
    pos = 0

    return id # IN CASE CALLER WANTS TEMP ID WE MAY HAVE ASSIGNED

def read_fasta(ifile):
    "iterate over id,title,seq from stream ifile"
    id = None
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and len(seq) > 0:
                yield id, title, seq
                isEmpty = False
            id = line[1:].split()[0]
            title = line[len(id)+2:]
            seq = ''
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
    if id is not None and len(seq) > 0:
        yield id, title, seq
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

def read_fasta_one_line(ifile): # @CTB deprecated; remove
    "read a single sequence line, return id,title,seq"
    id = None
    seq = ''
    while True:
        line = ifile.readline(1024) # READ AT MOST 1KB
        if line == '': # EOF
            break
        elif '>' == line[0]:
            id = line[1:].split()[0]
            title = line[len(id)+2:]
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
            if len(seq) > 0:
                return id, title, seq
    raise IOError('no readable sequence in FASTA file!')

def read_fasta_lengths(ifile):
    "Generate sequence ID,length from stream ifile"
    id = None
    seqLength = 0
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and seqLength > 0:
                yield id, seqLength
                isEmpty = False
            id = line[1:].split()[0]
            seqLength = 0
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seqLength += len(word)
    if id is not None and seqLength > 0:
        yield id, seqLength
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

class AATranslation(object):
    'customizable translation class'
    geneticCode = dict(TTY='F', TTR='L', TCN='S', TAY='Y', TGY='C', TGG='W',
                       CTN='L', CCN='P', CAY='H', CAR='Q', CGN='R',
                       ATY='I', ATA='I', ATG='M', ACN='T', AAY='N', AAR='K',
                       AGY='S', AGR='R',
                       GTN='V', GCN='A', GAY='D', GAR='E', GGN='G',
                       TAR='*', TGA='*')

    def __init__(self):
        'initialize our translation dictionary by applying N,Y,R codes'
        geneticCode = self.geneticCode.copy()
        for codon, aa in self.geneticCode.items():
            if codon[2] == 'N':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'G'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'Y':
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'R':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'G'] = aa
        self.geneticCode = geneticCode

    def __call__(self, s):
        'translate nucleotide string s to amino acid string'
        s = s.upper()
        s = s.replace('U', 'T')
        l = []
        for i in range(0, len(s), 3):
            try:
                l.append(self.geneticCode[s[i:i+3]])
            except KeyError:
                l.append('X') # uninterpretable
        return ''.join(l)

translate_orf = AATranslation() # default translation function


try:
                l.append(self.geneticCode[s[i:i+3]])
            except KeyError:
                l.append('X') # uninterpretable

DNA_SEQTYPE=0
RNA_SEQTYPE=1
PROTEIN_SEQTYPE=2

def guess_seqtype(s):
    dna_letters='AaTtUuGgCcNn'
    ndna=0
    nU=0
    nT=0
    for l in s:
        if l in dna_letters:
            ndna += 1
        if l=='U' or l=='u':
            nU += 1
        elif l=='T' or l=='t':
            nT += 1
    ratio=ndna/float(len(s))
    if ratio>0.85:
        if nT>nU:
            return DNA_SEQTYPE
        else:
            return RNA_SEQTYPE
    else:
        return PROTEIN_SEQTYPE

seq_id_counter=0

def new_seq_id():
    global seq_id_counter
    seq_id_counter += 1
    return str(seq_id_counter-1)

def write_fasta(ofile, s, chunk=60, id=None, reformatter=None):
    "Trivial FASTA output"
    if id is None:
        try:
            id = str(s.id)
        except AttributeError:
            id = new_seq_id()

    ofile.write('>' + id + '\n')
    seq = str(s)
    if reformatter is not None: # APPLY THE DESIRED REFORMATTING
        seq = reformatter(seq)
    end = len(seq)
    pos = 0
    while 1:
        ofile.write(seq[pos:pos+chunk] + '\n')
        pos += chunk
        if pos >= end:
            break
    return id # IN CASE CALLER WANTS TEMP ID WE MAY HAVE ASSIGNED

def read_fasta(ifile):
    "iterate over id,title,seq from stream ifile"
    id = None
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and len(seq) > 0:
                yield id, title, seq
                isEmpty = False
            id = line[1:].split()[0]
            title = line[len(id)+2:]
            seq = ''
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
    if id is not None and len(seq) > 0:
        yield id, title, seq
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

def read_fasta_one_line(ifile): # @CTB deprecated; remove
    "read a single sequence line, return id,title,seq"
    id = None
    seq = ''
    while True:
        line = ifile.readline(1024) # READ AT MOST 1KB
        if line == '': # EOF
            break
        elif '>' == line[0]:
            id = line[1:].split()[0]
            title = line[len(id)+2:]
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seq += word
            if len(seq) > 0:
                return id, title, seq
    raise IOError('no readable sequence in FASTA file!')

def read_fasta_lengths(ifile):
    "Generate sequence ID,length from stream ifile"
    id = None
    seqLength = 0
    isEmpty = True
    for line in ifile:
        if '>' == line[0]:
            if id is not None and seqLength > 0:
                yield id, seqLength
                isEmpty = False
            id = line[1:].split()[0]
            seqLength = 0
        elif id is not None: # READ SEQUENCE
            for word in line.split(): # GET RID OF WHITESPACE
                seqLength += len(word)
    if id is not None and seqLength > 0:
        yield id, seqLength
    elif isEmpty:
        raise IOError('no readable sequence in FASTA file!')

class AATranslation(object):
    'customizable translation class'
    geneticCode = dict(TTY='F', TTR='L', TCN='S', TAY='Y', TGY='C', TGG='W',
                       CTN='L', CCN='P', CAY='H', CAR='Q', CGN='R',
                       ATY='I', ATA='I', ATG='M', ACN='T', AAY='N', AAR='K',
                       AGY='S', AGR='R',
                       GTN='V', GCN='A', GAY='D', GAR='E', GGN='G',
                       TAR='*', TGA='*')

    def __init__(self):
        'initialize our translation dictionary by applying N,Y,R codes'
        geneticCode = self.geneticCode.copy()
        for codon, aa in self.geneticCode.items():
            if codon[2] == 'N':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'G'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'Y':
                geneticCode[codon[:2]+'T'] = aa
                geneticCode[codon[:2]+'C'] = aa
            elif codon[2] == 'R':
                geneticCode[codon[:2]+'A'] = aa
                geneticCode[codon[:2]+'G'] = aa
        self.geneticCode = geneticCode

    def __call__(self, s):
        'translate nucleotide string s to amino acid string'
        s = s.upper()
        s = s.replace('U', 'T')
        l = []
        for i in range(0, len(s), 3):

        return ''.join(l)

translate_orf = AATranslation() # default translation function


self.check_unary('trunc')
        self.check_unary_complex_unsupported('trunc')

import unittest

import numpy
import pytest

import dpnp as cupy
from tests.third_party.cupy import testing

@testing.gpu
class TestRounding(unittest.TestCase):

    @testing.for_all_dtypes(no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary(self, name, xp, dtype):
        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_complex(self, name, xp, dtype):
        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    def check_unary_complex_unsupported(self, name, dtype):
        for xp in (numpy, cupy):
            a = testing.shaped_arange((2, 3), xp, dtype)
            with pytest.raises(TypeError):
                getattr(xp, name)(a)

    @testing.for_dtypes(['?', 'b', 'h', 'i', 'q', 'e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_negative(self, name, xp, dtype):
        a = xp.array([-3, -2, -1, 1, 2, 3], dtype=dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_negative_complex(self, name, xp, dtype):
        a = xp.array([-3 - 3j, -2 - 2j, -1 - 1j, 1 + 1j, 2 + 2j, 3 + 3j], dtype=dtype)
        return getattr(xp, name)(a)

    def test_rint(self):
        self.check_unary('rint')
        self.check_unary_complex('rint')

    def test_rint_negative(self):
        self.check_unary_negative('rint')
        self.check_unary_negative_complex('rint')

    def test_floor(self):
        self.check_unary('floor')
        self.check_unary_complex_unsupported('floor')

    def test_ceil(self):
        self.check_unary('ceil')
        self.check_unary_complex_unsupported('ceil')

    def test_trunc(self):

    def test_fix(self):
        self.check_unary('fix')
        self.check_unary_complex_unsupported('fix')

    def test_around(self):
        self.check_unary('around')
        self.check_unary_complex('around')

    def test_round_(self):
        self.check_unary('round_')
        self.check_unary_complex('around')

@testing.parameterize(*testing.product({
    'decimals': [-2, -1, 0, 1, 2],
}))
class TestRound(unittest.TestCase):

    shape = (20,)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_round(self, xp, dtype):
        if dtype == numpy.bool_:
            # avoid cast problem
            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
            return xp.around(a, 0)
        if dtype == numpy.float16:
            # avoid accuracy problem
            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
            return xp.around(a, 0)
        a = testing.shaped_random(self.shape, xp, scale=100, dtype=dtype)
        return xp.around(a, self.decimals)

    @testing.numpy_cupy_array_equal()
    def test_round_out(self, xp):
        a = testing.shaped_random(self.shape, xp, scale=100, dtype='d')
        out = xp.empty_like(a)
        xp.around(a, self.decimals, out)
        return out

@testing.parameterize(*testing.product({
    'decimals': [-100, -99, -90, 0, 90, 99, 100],
}))
class TestRoundExtreme(unittest.TestCase):

    shape = (20,)

    @testing.for_dtypes([numpy.float64, numpy.complex128])
    @testing.numpy_cupy_allclose()
    def test_round_large(self, xp, dtype):
        a = testing.shaped_random(self.shape, xp, scale=1e100, dtype=dtype)
        return xp.around(a, self.decimals)

    @testing.for_dtypes([numpy.float64, numpy.complex128])
    @testing.numpy_cupy_allclose()
    def test_round_small(self, xp, dtype):
        a = testing.shaped_random(self.shape, xp, scale=1e-100, dtype=dtype)
        return xp.around(a, self.decimals)

@testing.parameterize(*testing.product({
    'value': [
        (14, -1),
        (15, -1),
        (16, -1),
        (14.0, -1),
        (15.0, -1),
        (16.0, -1),
        (1.4, 0),
        (1.5, 0),
        (1.6, 0),
    ]
}))
class TestRoundBorder(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_positive1(self, xp):
        a, decimals = self.value
        return xp.around(a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_positive2(self, xp):
        a, decimals = self.value
        a = xp.asarray(a)
        return xp.around(a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_negative1(self, xp):
        a, decimals = self.value
        return xp.around(-a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_negative2(self, xp):
        a, decimals = self.value
        a = xp.asarray(a)
        return xp.around(-a, decimals)


import unittest

import numpy
import pytest

import dpnp as cupy
from tests.third_party.cupy import testing

@testing.gpu
class TestRounding(unittest.TestCase):

    @testing.for_all_dtypes(no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary(self, name, xp, dtype):
        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_complex(self, name, xp, dtype):
        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    def check_unary_complex_unsupported(self, name, dtype):
        for xp in (numpy, cupy):
            a = testing.shaped_arange((2, 3), xp, dtype)
            with pytest.raises(TypeError):
                getattr(xp, name)(a)

    @testing.for_dtypes(['?', 'b', 'h', 'i', 'q', 'e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_negative(self, name, xp, dtype):
        a = xp.array([-3, -2, -1, 1, 2, 3], dtype=dtype)
        return getattr(xp, name)(a)

    @testing.for_complex_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def check_unary_negative_complex(self, name, xp, dtype):
        a = xp.array([-3 - 3j, -2 - 2j, -1 - 1j, 1 + 1j, 2 + 2j, 3 + 3j], dtype=dty

        return getattr(xp, name)(a)

    def test_rint(self):
        self.check_unary('rint')
        self.check_unary_complex('rint')

    def test_rint_negative(self):
        self.check_unary_negative('rint')
        self.check_unary_negative_complex('rint')

    def test_floor(self):
        self.check_unary('floor')
        self.check_unary_complex_unsupported('floor')

    def test_ceil(self):
        self.check_unary('ceil')
        self.check_unary_complex_unsupported('ceil')

    def test_trunc(self):
        self.check_unary('trunc')
        self.check_unary_complex_unsupported('trunc')

    def test_fix(self):
        self.check_unary('fix')
        self.check_unary_complex_unsupported('fix')

    def test_around(self):
        self.check_unary('around')
        self.check_unary_complex('around')

    def test_round_(self):
        self.check_unary('round_')
        self.check_unary_complex('around')

@testing.parameterize(*testing.product({
    'decimals': [-2, -1, 0, 1, 2],
}))
class TestRound(unittest.TestCase):

    shape = (20,)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_round(self, xp, dtype):
        if dtype == numpy.bool_:
            # avoid cast problem
            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
            return xp.around(a, 0)
        if dtype == numpy.float16:
            # avoid accuracy problem
            a = testing.shaped_random(self.shape, xp, scale=10, dtype=dtype)
            return xp.around(a, 0)
        a = testing.shaped_random(self.shape, xp, scale=100, dtype=dtype)
        return xp.around(a, self.decimals)

    @testing.numpy_cupy_array_equal()
    def test_round_out(self, xp):
        a = testing.shaped_random(self.shape, xp, scale=100, dtype='d')
        out = xp.empty_like(a)
        xp.around(a, self.decimals, out)
        return out

@testing.parameterize(*testing.product({
    'decimals': [-100, -99, -90, 0, 90, 99, 100],
}))
class TestRoundExtreme(unittest.TestCase):

    shape = (20,)

    @testing.for_dtypes([numpy.float64, numpy.complex128])
    @testing.numpy_cupy_allclose()
    def test_round_large(self, xp, dtype):
        a = testing.shaped_random(self.shape, xp, scale=1e100, dtype=dtype)
        return xp.around(a, self.decimals)

    @testing.for_dtypes([numpy.float64, numpy.complex128])
    @testing.numpy_cupy_allclose()
    def test_round_small(self, xp, dtype):
        a = testing.shaped_random(self.shape, xp, scale=1e-100, dtype=dtype)
        return xp.around(a, self.decimals)

@testing.parameterize(*testing.product({
    'value': [
        (14, -1),
        (15, -1),
        (16, -1),
        (14.0, -1),
        (15.0, -1),
        (16.0, -1),
        (1.4, 0),
        (1.5, 0),
        (1.6, 0),
    ]
}))
class TestRoundBorder(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_positive1(self, xp):
        a, decimals = self.value
        return xp.around(a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_positive2(self, xp):
        a, decimals = self.value
        a = xp.asarray(a)
        return xp.around(a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_negative1(self, xp):
        a, decimals = self.value
        return xp.around(-a, decimals)

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_around_negative2(self, xp):
        a, decimals = self.value
        a = xp.asarray(a)
        return xp.around(-a, decimals)


kde4.install()

    pisitools.dodoc("NEWS", "README")

    pisitools.remove("/usr/share/locale/*/LC_MESSAGES/libkipi.mo")

from pisi.actionsapi import shelltools
from pisi.actionsapi import pisitools
from pisi.actionsapi import kde4
from pisi.actionsapi import get
from pisi.actionsapi import cmaketools

def setup():
    kde4.configure("-DCMAKE_INSTALL_PREFIX=/usr \
                    -DCMAKE_BUILD_TYPE=Release \
                    -DFORCED_UNBUNDLE=ON \
                    -DWITH_LQR=ON \
                    -DWITH_LENSFUN=ON \
                    -DWITH_MarbleWidget=ON \
                    -DENABLE_LCMS2=ON \
                    -DDIGIKAMSC_USE_PRIVATE_KDEGRAPHICS=OFF \
                    -DDIGIKAMSC_USE_PRIVATE_SHAREDLIBS=ON \
                    -DDIGIKAMSC_COMPILE_LIBKGEOMAP=ON \
                    -DDIGIKAMSC_COMPILE_LIBKVKONTAKTE=ON \
                    -DDIGIKAMSC_COMPILE_LIBMEDIAWIKI=ON \
                    -DDIGIKAMSC_COMPILE_LIBKFACE=ON \
                    -DDIGIKAMSC_COMPILE_LIBKIPI=ON ")

def build():
    kde4.make()

def install():


from pisi.actionsapi import shelltools
from pisi.actionsapi import pisitools
from pisi.actionsapi import kde4
from pisi.actionsapi import get
from pisi.actionsapi import cmaketools

def setup():
    kde4.configure("-DCMAKE_INSTALL_PREFIX=/usr \
                    -DCMAKE_BUILD_TYPE=Release \
                    -DFORCED_UNBUNDLE=ON \
                    -DWITH_LQR=ON \
                    -DWITH_LENSFUN=ON \
                    -DWITH_MarbleWidget=ON \
                    -DENABLE_LCMS2=ON \
                    -DDIGIKAMSC_USE_PRIVATE_KDEGRAPHICS=OFF \
                    -DDIGIKAMSC_USE_PRIVATE_SHAREDLIBS=ON \
                    -DDIGIKAMSC_COMPILE_LIBKGEOMAP=ON \
                    -DDIGIKAMSC_COMPILE_LIBKVKONTAKTE=ON \
                    -DDIGIKAMSC_COMPILE_LIBMEDIAWIKI=ON \
                    -DDIGIKAMSC_COMPILE_LIBKFACE=ON \
                    -DDIGIKAMSC_COMPILE_LIBKIPI=ON ")

def build():
    kde4.make()

def install():
    kde4.install

    pisitools.dodoc("NEWS", "README")

    pisitools.remove("/usr/share/locale/*/LC_MESSAGES/libkipi.mo")


# optimization pass to promote a scalar/vector DAG into vector registers

# This file is part of metalibm (http://example.com)

from metalibm_core.core.ml_formats import (
    ML_Custom_FixedPoint_Format
)
from metalibm_core.core.ml_hdl_format import (
    RTL_FixedPointFormat, ML_StdLogicVectorFormat
)
import metalibm_core.core.ml_hdl_format as ml_hdl_format

from metalibm_core.core.ml_table import ML_TableFormat

from metalibm_core.core.ml_operations import (
    is_leaf_node
)
from metalibm_core.core.passes import FunctionPass, METALIBM_PASS_REGISTER

from metalibm_core.utility.log_report import Log

# high verbosity log-level for expand_multi_precision pass module
LOG_LEVEL_LEGALIZE_RTL2SW = Log.LogLevel("LegalizeRTL2SWFormat")

class Pass_ExhaustiveSearch(FunctionPass):
    """ Check precision of every node appearing in the graph """
    pass_tag = "exhaustive_search"
    def __init__(self, tag, target):

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """
        raise NotImplementedError

    def execute_on_optree(self, optree, fct, fct_group, memoization_map):
        if optree in memoization_map:
            return memoization_map[optree]
        if not is_leaf_node(optree):
            for op in optree.inputs:
                _ = self.execute_on_optree(op, fct, fct_group, memoization_map)
        result = self.execute_on_node(optree)
        memoization_map[optree] = result
        return result

SW_StdLogic = ML_Custom_FixedPoint_Format(1, 0, signed=False)

def legalize_rtl_to_sw_format(node_format):
    new_format = node_format
    if node_format is ml_hdl_format.ML_StdLogic:
        new_format = SW_StdLogic
    elif isinstance(node_format, RTL_FixedPointFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_integer_size(),
            node_format.get_frac_size(),
            signed=node_format.get_signed()
        )
    elif isinstance(node_format, ML_StdLogicVectorFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_bit_size(),
            0,
            signed=False)
    return new_format

@METALIBM_PASS_REGISTER
class Pass_LegalizeRTLtoSWFortmat(Pass_ExhaustiveSearch):
    pass_tag = "legalizertl2swformat"
    def __init__(self, target):
        Pass_ExhaustiveSearch.__init__(self, "legalize RTL to SW compatible format", target)

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """
        node_format = node.get_precision()
        new_format = None
        if node_format is ml_hdl_format.ML_StdLogic:
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_TableFormat):
            storage_format = node_format.get_storage_precision()
            new_storage_format = legalize_rtl_to_sw_format(storage_format)
            node_format.set_storage_precision(new_storage_format)
            Log.report(Log.Info, "translating RTL storage format from {} to {}".format(storage_format, new_storage_format))
            Log.report(Log.Info, "translating RTL table format to {}, {}".format(node.get_precision(), node.get_precision().get_storage_precision()))
        elif isinstance(node_format, RTL_FixedPointFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_StdLogicVectorFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        if not new_format is None:
            Log.report(LOG_LEVEL_LEGALIZE_RTL2SW, "translating RTL format {} to {}".format(node_format, new_format))
            node.set_precision(new_format)
        return new_format


# optimization pass to promote a scalar/vector DAG into vector registers

# This file is part of metalibm (http://example.com)

from metalibm_core.core.ml_formats import (
    ML_Custom_FixedPoint_Format
)
from metalibm_core.core.ml_hdl_format import (
    RTL_FixedPointFormat, ML_StdLogicVectorFormat
)
import metalibm_core.core.ml_hdl_format as ml_hdl_format

from metalibm_core.core.ml_table import ML_TableFormat

from metalibm_core.core.ml_operations import (
    is_leaf_node
)
from metalibm_core.core.passes import FunctionPass, METALIBM_PASS_REGISTER

from metalibm_core.utility.log_report import Log

# high verbosity log-level for expand_multi_precision pass module
LOG_LEVEL_LEGALIZE_RTL2SW = Log.LogLevel("LegalizeRTL2SWFormat")

class Pass_ExhaustiveSearch(FunctionPass):
    """ Check precision of every node appearing in the graph """
    pass_tag = "exhaustive_search"
    def __init__(self, tag, target):
        FunctionPass.__init__(self, tag, target)

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """
        raise NotImplementedError

    def execute_on_optree(self, optree, fct, fct_group, memoization_map):
        if optree in memoization_map:
            return memoization_map[optree]
        if not is_leaf_node(optree):
            for op in optree.inputs:
                _ = self.execute_on_optree(op, fct, fct_group, memoization_map)
        result = self.execute_on_node(optree)
        memoization_map[optree] = result
        return result

SW_StdLogic = ML_Custom_FixedPoint_Format(1, 0, signed=False)

def legalize_rtl_to_sw_format(node_format):
    new_format = node_format
    if node_format is ml_hdl_format.ML_StdLogic:
        new_format = SW_StdLogic
    elif isinstance(node_format, RTL_FixedPointFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_integer_size(),
            node_format.get_frac_size(),
            signed=node_format.get_signed()
        )
    elif isinstance(node_format, ML_StdLogicVectorFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_bit_size(),
            0,
            signed=False)
    return new_format

@METALIBM_PASS_REGISTER
class Pass_LegalizeRTLtoSWFortmat(Pass_ExhaustiveSearch):
    pass_tag = "legalizertl2swformat"
    def __init__(self, target):
        Pass_ExhaustiveSearch.__init__(self, "legalize RTL to SW compatible format", target)

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """
        node_format = node.get_precision()
        ne

        if node_format is ml_hdl_format.ML_StdLogic:
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_TableFormat):
            storage_format = node_format.get_storage_precision()
            new_storage_format = legalize_rtl_to_sw_format(storage_format)
            node_format.set_storage_precision(new_storage_format)
            Log.report(Log.Info, "translating RTL storage format from {} to {}".format(storage_format, new_storage_format))
            Log.report(Log.Info, "translating RTL table format to {}, {}".format(node.get_precision(), node.get_precision().get_storage_precision()))
        elif isinstance(node_format, RTL_FixedPointFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_StdLogicVectorFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        if not new_format is None:
            Log.report(LOG_LEVEL_LEGALIZE_RTL2SW, "translating RTL format {} to {}".format(node_format, new_format))
            node.set_precision(new_format)
        return new_format


# optimization pass to promote a scalar/vector DAG into vector registers

# This file is part of metalibm (http://example.com)

from metalibm_core.core.ml_formats import (
    ML_Custom_FixedPoint_Format
)
from metalibm_core.core.ml_hdl_format import (
    RTL_FixedPointFormat, ML_StdLogicVectorFormat
)
import metalibm_core.core.ml_hdl_format as ml_hdl_format

from metalibm_core.core.ml_table import ML_TableFormat

from metalibm_core.core.ml_operations import (
    is_leaf_node
)
from metalibm_core.core.passes import FunctionPass, METALIBM_PASS_REGISTER

from metalibm_core.utility.log_report import Log

# high verbosity log-level for expand_multi_precision pass module
LOG_LEVEL_LEGALIZE_RTL2SW = Log.LogLevel("LegalizeRTL2SWFormat")

class Pass_ExhaustiveSearch(FunctionPass):
    """ Check precision of every node appearing in the graph """
    pass_tag = "exhaustive_search"
    def __init__(self, tag, target):
        FunctionPass.__init__(self, tag, target)

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """

    def execute_on_optree(self, optree, fct, fct_group, memoization_map):
        if optree in memoization_map:
            return memoization_map[optree]
        if not is_leaf_node(optree):
            for op in optree.inputs:
                _ = self.execute_on_optree(op, fct, fct_group, memoization_map)
        result = self.execute_on_node(optree)
        memoization_map[optree] = result
        return result

SW_StdLogic = ML_Custom_FixedPoint_Format(1, 0, signed=False)

def legalize_rtl_to_sw_format(node_format):
    new_format = node_format
    if node_format is ml_hdl_format.ML_StdLogic:
        new_format = SW_StdLogic
    elif isinstance(node_format, RTL_FixedPointFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_integer_size(),
            node_format.get_frac_size(),
            signed=node_format.get_signed()
        )
    elif isinstance(node_format, ML_StdLogicVectorFormat):
        new_format = ML_Custom_FixedPoint_Format(
            node_format.get_bit_size(),
            0,
            signed=False)
    return new_format

@METALIBM_PASS_REGISTER
class Pass_LegalizeRTLtoSWFortmat(Pass_ExhaustiveSearch):
    pass_tag = "legalizertl2swformat"
    def __init__(self, target):
        Pass_ExhaustiveSearch.__init__(self, "legalize RTL to SW compatible format", target)

    def execute_on_node(self, node):
        """ execute pass action on node without input traversal """
        node_format = node.get_precision()
        new_format = None
        if node_format is ml_hdl_format.ML_StdLogic:
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_TableFormat):
            storage_format = node_format.get_storage_precision()
            new_storage_format = legalize_rtl_to_sw_format(storage_format)
            node_format.set_storage_precision(new_storage_format)
            Log.report(Log.Info, "translating RTL storage format from {} to {}".format(storage_format, new_storage_format))
            Log.report(Log.Info, "translating RTL table format to {}, {}".format(node.get_precision(), node.get_precision().get_storage_precision()))
        elif isinstance(node_format, RTL_FixedPointFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        elif isinstance(node_format, ML_StdLogicVectorFormat):
            new_format = legalize_rtl_to_sw_format(node_format)
        if not new_format is None:
            Log.report(LOG_LEVEL_LEGALIZE_RTL2SW, "translating RTL format {} to {}".format(node_format, new_format))
            node.set_precision(new_format)
        return new_format


"""Construct ScriptExporter"""
        e = self.exporter_class()

"""Tests for ScriptExporter"""

import os
import sys

from nbformat import v4
import nbconvert

from .base import ExportersTestsBase
from ..script import ScriptExporter

class TestScriptExporter(ExportersTestsBase):
    """Tests for ScriptExporter"""

    exporter_class = ScriptExporter

    def test_constructor(self):

    def test_export(self):
        """ScriptExporter can export something"""
        (output, resources) = self.exporter_class().from_filename(self._get_notebook())
        assert len(output) > 0

    def test_export_python(self):
        """delegate to custom exporter from language_info"""
        exporter = self.exporter_class()

        pynb = v4.new_notebook()
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertNotIn('# coding: utf-8', output)

        pynb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertIn('# coding: utf-8', output)

    def test_export_config_transfer(self):
        """delegate config to custom exporter from language_info"""
        nb = v4.new_notebook()
        nb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }

        exporter = self.exporter_class()
        exporter.from_notebook_node(nb)
        assert exporter._exporters['python'] != exporter
        assert exporter._exporters['python'].config == exporter.config

def test_script_exporter_entrypoint():
    nb = v4.new_notebook()
    nb.metadata.language_info = {
        'name': 'dummy',
        'mimetype': 'text/x-dummy',
    }

    p = os.path.join(os.path.dirname(nbconvert.tests.__file__), 'exporter_entrypoint')
    sys.path.insert(0, p)
    try:
        output, _ = ScriptExporter().from_notebook_node(nb)
        assert output == 'dummy-script-exported'
    finally:
        sys.path.remove(p)


"""Tests for ScriptExporter"""

import os
import sys

from nbformat import v4
import nbconvert

from .base import ExportersTestsBase
from ..script import ScriptExporter

class TestScriptExporter(ExportersTestsBase):
    """Tests for ScriptExporter"""

    exporter_class = ScriptExporter

    def test_constructor(self):
        """Construct ScriptExporter"""
        e = self.exporter_class()

    def test_export(self):
        """ScriptExporter can export something"""
        (output, resources) = self.exporter_class().from_filename(self._get_notebook())
        assert len(output) > 0

    def test_export_python(self):
        """delegate to custom exporter from language_info"""
        exporter = self.exporter_class()

        pynb = v4.new_notebook()
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertNotIn('# coding: utf-8', output)

        pynb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertIn('# coding: utf-8', output)

    def test_export_config_transfer(self):
        """delegate config to custom exporter from language_info"""
        nb = v4.new_notebook()
        nb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }

        exporter = self.exporter_class()
        exporter.from_notebook_node(nb)
        assert exporter._exporters['python'] != exporter
        assert exporter._exporters['python'].config == exporter.config

def test_script_exporter_entrypoint():
    nb = v4.new_notebook()
    nb.metadata.language_info = {
        'name': 'dummy',
        'mimetype': 'text/x-dummy',
    }

    p = os.path.join(os.path.dirname(nbconvert.tests.__file__), 'exporter_ent

    sys.path.insert(0, p)
    try:
        output, _ = ScriptExporter().from_notebook_node(nb)
        assert output == 'dummy-script-exported'
    finally:
        sys.path.remove(p)


"""Tests for ScriptExporter"""

import os
import sys

from nbformat import v4
import nbconvert

from .base import ExportersTestsBase
from ..script import ScriptExporter

class TestScriptExporter(ExportersTestsBase):
    """Tests for ScriptExporter"""

    exporter_class = ScriptExporter

    def test_constructor(self):
        """Construct ScriptExporter"""


try:
        output, _ = ScriptExporter().from_notebook_node(nb)
        assert output == 'dummy-script-exported'
    finally:
        sys.path.remove(p)

"""Tests for ScriptExporter"""

import os
import sys

from nbformat import v4
import nbconvert

from .base import ExportersTestsBase
from ..script import ScriptExporter

class TestScriptExporter(ExportersTestsBase):
    """Tests for ScriptExporter"""

    exporter_class = ScriptExporter

    def test_constructor(self):
        """Construct ScriptExporter"""
        e = self.exporter_class()

    def test_export(self):
        """ScriptExporter can export something"""
        (output, resources) = self.exporter_class().from_filename(self._get_notebook())
        assert len(output) > 0

    def test_export_python(self):
        """delegate to custom exporter from language_info"""
        exporter = self.exporter_class()

        pynb = v4.new_notebook()
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertNotIn('# coding: utf-8', output)

        pynb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }
        (output, resources) = self.exporter_class().from_notebook_node(pynb)
        self.assertIn('# coding: utf-8', output)

    def test_export_config_transfer(self):
        """delegate config to custom exporter from language_info"""
        nb = v4.new_notebook()
        nb.metadata.language_info = {
            'name': 'python',
            'mimetype': 'text/x-python',
            'nbconvert_exporter': 'python',
        }

        exporter = self.exporter_class()
        exporter.from_notebook_node(nb)
        assert exporter._exporters['python'] != exporter
        assert exporter._exporters['python'].config == exporter.config

def test_script_exporter_entrypoint():
    nb = v4.new_notebook()
    nb.metadata.language_info = {
        'name': 'dummy',
        'mimetype': 'text/x-dummy',
    }

    p = os.path.join(os.path.dirname(nbconvert.tests.__file__), 'exporter_entrypoint')
    sys.path.insert(0, p)


self.datasets = []
        self.title = title
        self.xlabel = xlabel
        self.ylabel = ylabel
        self.xmin = xmin
        self.xmax = xmax
        self.ymin = ymin
        self.ymax = ymax

# SecuML

import matplotlib.pyplot as plt

class LinePlot(object):

    def __init__(self, title=None, xlabel=None, ylabel=None, xmin=None,
                 xmax=None, ymin=None, ymax=None):

    def add_dataset(self, dataset):
        self.datasets.append(dataset)

    def to_png(self, filename):
        plt.clf()
        for dataset in self.datasets:
            if dataset.error_bars is not None:
                plt.errorbar(dataset.xvalues, dataset.values,
                             yerr=dataset.error_bars, label=dataset.label)
            else:
                plt.plot(dataset.xvalues, dataset.values, label=dataset.label)
        if self.xlabel is not None:
            plt.xlabel(self.xlabel)
        if self.ylabel is not None:
            plt.ylabel(self.ylabel)
        if self.title is not None:
            plt.title(self.title)
        plt.legend(loc='lower right')
        plt.savefig(filename)
        plt.clf()

    def to_pgf_plot(self, filename, standalone=False):
        with open(filename, 'w') as f:
            if standalone:
                f.write('\\documentclass[crop]{standalone}\n')
                f.write('\\usepackage{pgfplots}\n')
                f.write('\\usepackage{tikz}\n')
                f.write('\\begin{document}\n')
            f.write('\\begin{tikzpicture}\n'
                    '\t\\begin{axis} [\n')
            if self.xlabel is not None:
                f.write('\t\txlabel=\\large %s,\n' % self.xlabel)
            if self.ylabel is not None:
                f.write('\t\tylabel=\\large %s,\n' % self.ylabel)
            if self.title is not None:
                f.write('\t\ttitle = %s,\n' % self.title)
            for limit in ['xmin', 'xmax', 'ymin', 'ymax']:
                if getattr(self, limit) is not None:
                    f.write('\t\t%s = %f,\n' % (limit, getattr(self, limit)))
            f.write('\t\txlabel near ticks,\n'
                    '\t\tylabel near ticks,\n'
                    '\t\tlegend pos = south east,\n'
                    '\t\tlegend cell align = left\n'
                    '\t]\n')
            for dataset in self.datasets:
                f.write('\t\\addplot[color=%s, mark=none, ultra thick, '
                        'dashdotted] coordinates {\n' % dataset.label)
                for x, y in zip(dataset.xvalues, dataset.values):
                    f.write('\t\t(%f,%f)\n' % (x, y))
                f.write('\t};\n')
                f.write('\t\\definecolor{%s}{HTML}{%s}\n'
                        % (dataset.label, dataset.color[1:].upper()))
                f.write('\t\\addlegendentry{%s}\n' % dataset.label)
                if dataset.error_bars is not None:
                    f.write('\t\\addplot[color=%s, only marks, '
                            'error bars/y dir=both, error bars/y explicit, '
                            'mark options={%s}, forget plot]'
                            'coordinates {\n'
                            % (dataset.label, dataset.label))
                    for x, y, err in zip(dataset.xvalues, dataset.values,
                                         dataset.error_bars):
                        f.write('\t\t(%f,%f) +- (0,%f)\n' % (x, y, err))
                    f.write('\t};\n')
            f.write('\t\\end{axis}\n'
                    '\\end{tikzpicture}\n')
            if standalone:
                f.write('\\end{document}\n')