import os

BASE_DIR = os.path.expanduser("~")

LINUX = "LINUX"
LINUX_32 = LINUX + "32"
LINUX_64 = LINUX + "64"
MAC = "MAC"
WINDOWS = "WINDOWS"
WINDOWS_32 = WINDOWS + "32"
WINDOWS_64 = WINDOWS + "64"

class Setup(object):
  """Base class for installing prerequisites for Linux, Mac OS X and Windows.

  Attributes:
    bash_profile_changed: A boolean indicating whether or not the bash profile
        has been edited by the script, and the user should call source.
    cwebp_path: A string of the path to the location of the cwebp directory.
    cmake_path: A string of the path to the location of the cmake directory.

  Raises:
    BadDirectoryError: If a path given in the cwebp, cmake or ant flag does not
        exist.
  """

  def __init__(self, options):
    self.bash_profile_changed = False
    self.bash_profile = os.path.join(BASE_DIR, ".bash_profile") # Unused in
                                                                # Windows
    self.cwebp_path = os.path.join(BASE_DIR, options.cwebp_location)
    if not os.path.isdir(self.cwebp_path):
      raise BadDirectoryError("--cwebp", self.cwebp_path)
    self.cmake_path = os.path.join(BASE_DIR, options.cmake_location)
    if not os.path.isdir(self.cmake_path):
      raise BadDirectoryError("--cmake", self.cmake_path)
    self.ant_path = os.path.join(BASE_DIR, options.ant_location)
    if not os.path.isdir(self.ant_path):
      raise BadDirectoryError("--ant", self.ant_path)

  def has_bash_changed(self):
    """Returns wheter or not the bash profile has been changed."""

class SystemUnsupportedError(Exception):
  """Raised when an OS is unrecognised or unsupported."""

  def __init__(self, system):
    Exception.__init__(self)
    self.system = system

class VersionUnsupportedError(Exception):
  """Raised when the version of an OS is unrecognised or unsupported."""

  def __init__(self, version):
    Exception.__init__(self)
    self.version = version

class VersionTooHighError(Exception):
  """Raised when the OS version is greater than the highest supported."""

  def __init__(self, version):
    Exception.__init__(self)
    self.version = version

class VersionTooLowError(Exception):
  """Raised when the OS version is less than the lowest supported."""

  def __init__(self, version):
    Exception.__init__(self)
    self.version = version

class BadDirectoryError(Exception):
  """Raised when a directory given through a command line flag doesn't exist."""

  def __init__(self, flag, directory):
    Exception.__init__(self)
    self.flag = flag
    self.directory = directory

class InstallInterruptError(Exception):
  """Raised when installation of a program was interrupted by the user."""

  def __init__(self, program, instructions=""):
    Exception.__init__(self)
    self.program = program
    self.instructions = instructions

class InstallFailedError(Exception):
  """Raised when installation fails for reasons other than user interrupt."""

  def __init__(self, program, link="", instructions=""):
    Exception.__init__(self)
    self.program = program
    self.link = link
    self.instructions = instructions

class FileDownloadError(Exception):
  """Raised when a file was unable to download."""

  def __init__(self, link, instructions=""):
    Exception.__init__(self)
    self.link = link
    self.instructions = instructions

class UnknownFileTypeError(Exception):
  """Raised when the extension of a file is unrecognised."""

  def __init__(self, filetype, instructions):
    Exception.__init__(self)
    self.filetype = filetype
    self.instructions = instructions

class ExtractionError(Exception):
  """Raised when a compressed file was unable to be extracted."""

  def __init__(self, filepath):
    Exception.__init__(self)
    self.filepath = filepath

class CommandFailedError(Exception):
  """Raised when a subprocess fails for unforseeable reasons."""

  def __init__(self, command, link):
    Exception.__init__(self)
    self.command = command
    self.link = link

class PermissionDeniedError(Exception):
  """Raised when the script was not able to gain the correct permissions."""

  def __init__(self, program, instructions):
    Exception.__init__(self)
    self.program = program
    self.instructions = instructions

class WebbrowserFailedError(Exception):
  """Raised when a url was unable to be opened in the webbrowser."""

  def __init__(self, pagename, link):
    Exception.__init__(self)
    self.pagename = pagename
    self.link = link


        if isinstance(cidr, six.integer_types) or cidr.isdigit():
            n = Network.get(cidr)
        else:
            n = Network.get_by_name(cidr) or Network.get_by_netspace(cidr)
        return n and WrappedNetwork.from_macvlan(n) or None

import six
from netaddr import IPAddress
from werkzeug.security import gen_salt

from eru.agent import get_agent
from eru.ipam.base import BaseIPAM
from eru.ipam.structure import WrappedIP, WrappedNetwork
from eru.models.network import IP, Network
from eru.models.eip_pool import eip_pool

class MacVLANIPAM(BaseIPAM):

    def add_ip_pool(self, cidr, name):
        n = Network.create(name, cidr)
        return n and WrappedNetwork.from_macvlan(n) or None

    def remove_ip_pool(self, cidr):
        network = Network.get_by_netspace(cidr)
        if network:
            network.delete()

    def get_pool(self, cidr):
        """
        macvlan used to use name to identify.
        so here cidr may be a name string.
        and for compating, cidr can also be id.
        """

    def get_all_pools(self):
        networks = Network.list_networks()
        return [WrappedNetwork.from_macvlan(n) for n in networks if n]

    def allocate_ips(self, cidrs, container_id, spec_ips=None):
        """
        allocate ips for container_id, one ip per one cidr.
        if spec_ips is given, then the ip will be in spec_ips.
        note, for history reasons cidrs should always be given.
        """
        from eru.models.container import Container

        def _release_ips(ips):
            for ip in ips:
                ip.release()

        if not (cidrs or spec_ips):
            return True

        container = Container.get_by_container_id(container_id)
        count = len(container.ips.all())
        nstart = count+1 if count > 0 else 0
        nids = range(nstart, nstart+len(spec_ips or cidrs))

        networks = [Network.get_by_netspace(cidr) for cidr in cidrs]
        networks = [n for n in networks if n]

        if spec_ips:
            ips = [n.acquire_specific_ip(ip) for n, ip in zip(networks, spec_ips)]
        else:
            ips = [n.acquire_ip() for n in networks]

        ips = [i for i in ips if i]
        ip_dict = {ip.vlan_address: ip for ip in ips}

        agent = get_agent(container.host)
        ip_list = [(nid, ip.vlan_address) for nid, ip in zip(nids, ips)]

        resp = agent.add_container_vlan(container_id, gen_salt(8), ip_list)
        if not resp or resp.status_code != 200:
            _release_ips(ips)
            return False

        rv = resp.json()
        if not all(r['succ'] for r in rv):
            _release_ips(ips)
            return False

        for r in rv:
            ip = ip_dict.get(r['ip'], None)
            if ip:
                ip.set_vethname(r['veth'])
                ip.assigned_to_container(container)
        return True

    def reallocate_ips(self, container_id):
        """
        refresh ips on container_id.
        first, release all ips container occupied, then bind them as spec_ips.
        """
        from eru.models.container import Container

        container = Container.get_by_container_id(container_id)
        ips = container.ips.all()
        cidrs = [ip.network.netspace for ip in ips]
        spec_ips = [ip.address for ip in ips]

        for ip in ips:
            ip.release()

        return self.allocate_ips(cidrs, container_id, spec_ips)

    def get_ip_by_container(self, container_id):
        from eru.models.container import Container

        container = Container.get_by_container_id(container_id)
        if not container:
            return []
        return [WrappedIP.from_macvlan(i) for i in container.ips.all()]

    def release_ip_by_container(self, container_id):
        from eru.models.container import Container

        container = Container.get_by_container_id(container_id)
        if not container:
            return

        ips = IP.get_by_container(container.id)
        for ip in ips:
            ip.release()

    def add_eip(self, *eips):
        eips = [IPAddress(eip) for eip in eips]
        eip_pool.add_eip(*eips)

    def get_eip(self, eip=None):
        eip = eip and IPAddress(eip) or None
        return eip_pool.get_eip(eip)

    def release_eip(self, *eips):
        eips = [IPAddress(eip) for eip in eips]
        for eip in eips:
            eip_pool.release_eip(eip)


        self.start_date = start_date
        self.end_date = end_date
        self.metadata = metadata
        self.limit = limit
        self.random_sample = random_sample

from random import shuffle

class RecordingLookupProperties(object):
    def __init__(self, start_date, end_date=None, metadata=None, limit=None, random_sample=False):
        """
        :param start_date: Earliest date of recording
        :type start_date: datetime.datetime
        :param end_date: Latest date of recording
        :type end_date: datetime.datetime
        :param metadata: Optional metadata to filter recording by
        :type metadata: dict
        :param limit: Limit the number of recordings to fetch
        :type limit: int
        :param random_sample: True/False collect using random.shuffle (use random.seed to change selection)
        :type random_sample: boolean
        """

def find_matching_recording_ids(tape_recorder, category, lookup_properties):
    """
    :param tape_recorder: Tape cassette holding the recordings
    :type tape_recorder: playback.tape_recorder.TapeRecorder
    :param category: Recording category
    :type category: basestring
    :param lookup_properties: Recording lookup properties
    :type lookup_properties: RecordingLookupProperties
    :return: Iterator of recording ids based on lookup parameters
    :rtype: collections.Iterator[str]
    """
    recording_ids = tape_recorder.tape_cassette.iter_recording_ids(
            category, start_date=lookup_properties.start_date, end_date=lookup_properties.end_date,
            metadata=lookup_properties.metadata,
            limit=(None if lookup_properties.random_sample else lookup_properties.limit))

    if lookup_properties.random_sample:
        recording_ids = list(recording_ids)
        shuffle(recording_ids)
        recording_ids = iter(recording_ids[:lookup_properties.limit])

    return recording_ids


    if 3 != len( obj):
        raise TypeError( len( obj), "obj not 3-array")
    model, proj, view = readModProjView( model, proj, view)
    _glulib.gluUnProject( obj[ 0], obj[ 1], obj[ 2], model, proj, view,
                         win[0:1], win[ 1:2], win[ 2:9])

# copyright_notice

"""glu wrappers
"""
import numpy.oldnumeric as Numeric
from opengltk import util
from opengltk.extent import _gllib, _glulib
from opengltk.util import gltypmap, revtypmap ,\
     GLdouble, GLfloat, GLint, readModProjView
from opengltk.ccallback import swigptrcallback
import gl_wrapper

def gluBuild1DMipMaps( target, component, format, data):
    """
    target - GLenum
    component - GLint
    format - GLenum
    data - Numeric array
    """
    _glulib.gluBuild1DMipMaps( target, component, len( data), format,
                              gltypmap[ data.dtype.char], data)

def gluBuild2DMipMaps( target, component, width, format, data):
    """
    target - GLenum
    component - GLint
    width - GLsizei
    format - GLenum
    data - Numeric array
    """
    lendat = len( data)
    if lendat % width:
        raise TypeError( (width, len( data)),
                         'len( data) nota multiple of "width"')
    _glulib.gluBuild2DMipMaps( target, component, width, lendat/width, format,
                              gltypmap[ data.dtype.char], data)

def gluGetNurbsProperty( nurb, property):
    """
    nurb - GLUnurbs*
    property - GLenum
return - GLfloat
"""
    data = Numeric.zeros( 1, 'f')
    _glulib.gluGetNurbsProperty( nurb, property, data)
    return data[ 0]

def gluGetTessProperty( tess, property):
    """
    tess - GLUtesselator*
    property - GLenum
return - GLdouble
"""
    data = Numeric.zeros( 1, 'd')
    _glulib.gluGetTessProperty( nurb, property, data)
    return data[ 0]

def gluLoadSamplingMatrices( nurb, model, perspective, view):
    """
    nurb - GLUnurb
    model - sequence of 16 floats
    perspective - sequence of 16 floats
    view - sequence of 4 floats
    """
    model, perspective, view = readModProjView( model, perspective, view)
    _glulib.gluLoadSamplingMatrices( nurb, model, perspective, view)

def gluNurbsCallback( element, which, callback):
    return _glulib.gluNurbsCallback( element, which,
                                    swigptrcallback( util,
                                                     ('void', ['GLenum']),
                                                     callback))

def gluNurbsCurve( nurb, type, knots, order, control, stride):
    """
    nurb - GLUnurbs
    type - GLenum
    knots - seq( GLfloat)
    order - GLint
    control - seq( GLfloat)
    stride - GLint
    """

    lknot = len( knots)
    assert stride
    if lknot % stride:
        raise TypeError( (len( knots), stride),
                         "(len( knots), stride) not compatible")
    nknot = lknot / stride
    if len( control) != nknot - order:
        raise TypeError( (len( knots), order, stride, len( control)),
                         '(len( knots), order, stride, len( control))'
                         ' not compatible')

    _glulib.gluNurbsCurve( nurb, nknot, knots, stride,
                          control, order, type)

def gluNurbsSurface( nurb, type, sKnots, tKnots, sOrder, tOrder, control,
                     sStride, tStride):
    """
    nurb - GLUnurbs
    type - GLenum
    sKnots, tKnots - seq( GLfloat)
    sOrder, tOrder - GLint
    control - seq( GLfloat)
    sStride, tStride - GLint
    """
    assert sStride
    assert tStride

    lsknot = len( sKnots)
    if lsknot % sStride:
        raise TypeError( (len( sKnots), sStride),
                         "(len( sKnots), sStride) not compatible")
    nsknot = lsknot / sStride

    ltknot = len( tKnots)
    if ltknot % sStride:
        raise TypeError( (len( tKnots), sStride),
                         "(len( tKnots), sStride) not compatible")
    ntknot = ltknot / sStride

    if len( control) != (nsknot - sOrder)*(ntknot - tOrder):
        raise TypeError( (len( control), (nsknot, sOrder), (ntknot, tOrder)),
                         '(len( control), (nsknot, sOrder), (ntknot, tOrder))'
                         ' not compatible')

    _glulib.gluNurbsSurface( nurb, nsknot, sKnots, ntknot, tKnots,
                            sStride, tStride, control, sOrder, tOrder, type)

def gluPickMatrix( x, y, delX, delY, view):
    """
    x, y, delX, delY - GLdouble
    """
    if len( view) != 4:
        raise TypeError( view, len( view), '4 expected')
    if not isinstance(view , Numeric.ArrayType):
        view = Numeric.array(view, GLint)
    _glulib.gluPickMatrix( x, y, delX, delY, view)

def gluProject( obj, model, proj, view):
    """
    obj - seq( 3, GLdouble)
    model - seq( gl.MODELVIEW_MATRIX, GLfloat)
    perspective - seq( gl.PROJECTION_MATRIX, GLfloat)
    view - seq( gl.VIEWPORT, GLfloat)
    return - Numeric array (shape: (3,), type: GLdouble)
    """
    win = Numeric.zeros( 3, GLdouble)
    gluProjectv( obj, model, proj, view, win[0:1], win[ 1:2], win[ 2:3])
    return win

def gluProjectv( obj, model, proj, view, win):
    """
    obj - seq( 3, GLdouble)
    model - seq( gl.MODELVIEW_MATRIX, GLfloat)
    perspective - seq( gl.PROJECTION_MATRIX, GLfloat)
    view - seq( gl.VIEWPORT, GLfloat)
    win - Numeric array (shape: (3,), type: GLdouble): gets the new coordinates
    """

    if 3 != len( obj):
        raise TypeError( len( obj), "obj not 3-array")
    model, proj, view = readModProjView( model, proj, view)
    _glulib.gluProject( obj[ 0], obj[ 1], obj[ 2], model, proj, view,
                     win[0:1], win[ 1:2], win[ 2:3])

def gluPwlCurve( nurb, type, data, stride):
    """
    nurb - GLUnurbs*
    type - GLenum
    data - seq( GLfloat)
    stride - GLint
    """
    assert stride
    ldata = len( data)
    if ldata % stride:
        raise TypeError( (len( data), stride),
                         "len( data) not a multiple of stride")
    _glulib.gluPwlCurve( nurb, len( data) / stride, data, stride, type)

def gluQuadricCallback( element, which, callback):
    return _glulib.gluQuadricCallback( element, which,
                                      swigptrcallback( util,
                                                       ('void', ['GLenum']),
                                                       callback))

def gluScaleImage( format, wIn, dataIn, wOut, hOut, typeOut):
    """
    format - GLenum
    wIn - GLsizei
    dataIn - Numeric array / list
    wOut, hOut - GLsizei
    typeOut - GLenum
    return - Numeric array(shape: wOut*hOut, type; typeOut)
    """
    lin = len( dataIn)
    if lin % wIn:
        raise TypeError( (len( dataIn), wIn),
                         "len( dataIn) not multiple of wIn")

    gltout = typeOut
    typeOut = revtypmap[ typeout]
    result = Numeric.zeros( wOut * hOut, typeOut)
    _glulib.gluScaleImage( format,
                          wIn, lin / wIn, gltypmap[ dataIn.dtype.char], dataIn,
                          wOut, hOut, gltout, result)
    return result

def gluTessCallback( element, which, callback):
    return _glulib.gluTessCallback( element, which,
                                   swigptrcallback( util,
                                                    ('void', ['GLenum']),
                                                    callback))

def gluTessNormaldv( tess, normale):
    """
    tess - GLUtesselator*
    normale - seq( 3, GLdouble)
    """
    _glulib.gluTessNormal( tess, normale[ 0], normale[ 1], normale[ 2])

def gluTessNormalfv( tess, normale):
    """
    tess - GLUtesselator*
    normale - seq( 3, GLfloat)
    """
    _glulib.gluTessNormal( tess, normale[ 0], normale[ 1], normale[ 2])

def gluTessVertex( tess, location, data):
    """
    tess - GLUtesselator*
    location - seq( 3, GLdouble)
    data - GLvoid*
    """
    if 3 != len( location):
        raise TypeError( len( location), "location not a 3-array")
    _glulib.gluTessVertex( tess, location, data)

def gluUnProject( obj, model, proj, view):
    """
    obj - seq( 3, GLdouble)
    model - seq( gl.MODELVIEW_MATRIX, GLfloat)
    perspective - seq( gl.PROJECTION_MATRIX, GLfloat)
    view - seq( gl.VIEWPORT, GLfloat)
return - Numeric.array ( 3, GLdouble)
    """
    win = Numeric.zeros( 3, GLdouble)
    gluUnProjectv( obj, model, proj, view, win )
    return win[:3]

def gluUnProjectv( obj, model, proj, view, win):
    """
    obj - seq( 3, GLdouble)
    model - seq( gl.MODELVIEW_MATRIX, GLfloat)
    perspective - seq( gl.PROJECTION_MATRIX, GLfloat)
    view - seq( gl.VIEWPORT, GLfloat)
    win - Numeric array( 3, GLdouble): gets the new coordinates
    """

__all__ = [
    'gluBuild1DMipMaps',
    'gluBuild2DMipMaps',
    'gluGetNurbsProperty',
    'gluGetTessProperty',
    'gluLoadSamplingMatrices',
    'gluNurbsCallback',
    'gluNurbsCurve',
    'gluNurbsSurface',
    'gluPickMatrix',
    'gluProject',
    'gluProjectv',
    'gluPwlCurve',
    'gluQuadricCallback',
    'gluScaleImage',
    'gluTessCallback',
    'gluTessNormaldv',
    'gluTessNormalfv',
    'gluTessVertex',
    'gluUnProject',
    'gluUnProjectv',
    ]


import os
import os.path
import shutil
import time
import tqdm
import sqlalchemy.exc

import RawArchiver.RawActiveModules
import RawArchiver.RawEngine

import common.database as db
import common.Exceptions
import common.management.file_cleanup
import common.global_constants
import common.util.urlFuncs as urlFuncs

from config import C_RAW_RESOURCE_DIR
from sqlalchemy_continuum_vendored.utils import version_table

def exposed_purge_raw_invalid_urls():
	'''
	Delete all raw-archiver rows that aren't
	attached to a archiver module.
	'''

	sess1 = db.get_db_session(postfix='iter_sess')
	sess2 = db.get_db_session(postfix='delete_sess')

	print("Loading files from database...")

	est = sess1.execute("SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='raw_web_pages';")
	res = est.scalar()
	print("Estimated row-count: %s" % res)

	last_bad = ""
	last_commit = 0
	deleted = 0
	maxlen = 0
	changed_rows = 0
	total_rows = 0
	with tqdm.tqdm(total=res) as pbar:
		bad = 0
		for row in sess1.query(db.RawWebPages).yield_per(1000):
			modules_wants_url = any([mod.cares_about_url(row.url) for mod in RawArchiver.RawActiveModules.ACTIVE_MODULES])
			has_badwords      = any([badword in row.url for badword in common.global_constants.GLOBAL_BAD_URLS])
			if not modules_wants_url or has_badwords:
				last_bad = row.netloc
				print("Unwanted: ", row.url)

				changed_rows = sess2.query(db.RawWebPages) \
					.filter(db.RawWebPages.url == row.url) \
					.delete(synchronize_session=False)

				bad += 1
				deleted += 1

			total_rows += 1
			if bad > 5000:
				bad = 0
				last_commit = deleted
				sess2.commit()
			else:
				msg = "Deleted: %s, since commit: %s, last_bad: '%s' (%s, %s%%)" % \
					(deleted, deleted-last_commit, last_bad, changed_rows, 100.0*(deleted / total_rows))

				maxlen = max(len(msg), maxlen)
				pbar.set_description(msg.ljust(maxlen), refresh=False)
			pbar.update(n=1)

	sess1.commit()
	sess2.commit()

def exposed_purge_raw_invalid_urls_from_history():
	'''
	Delete all raw-archiver rows that aren't
	attached to a archiver module.
	'''

	sess1 = db.get_db_session(postfix='iter_sess')
	sess2 = db.get_db_session(postfix='delete_sess')

	ctbl = version_table(db.RawWebPages.__table__)

	print("Loading files from database...")

	est = sess1.execute("SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='raw_web_pages_version';")
	res = est.scalar()
	print("Estimated row-count: %s" % res)

	last_bad = ""
	deleted = 0
	total_rows = 0
	last_commit = 0
	maxlen = 0
	changed_rows = 0
	with tqdm.tqdm(total=res) as pbar:
		bad = 0

		for rurl, rnetloc in sess1.query(ctbl.c.url, ctbl.c.netloc).yield_per(1000):
			modules_wants_url = any([mod.cares_about_url(rurl) for mod in RawArchiver.RawActiveModules.ACTIVE_MODULES])
			has_badwords      = any([badword in rurl for badword in common.global_constants.GLOBAL_BAD_URLS])
			if not modules_wants_url or has_badwords:
				last_bad = rnetloc

				changed_rows = sess2.query(ctbl) \
					.filter(ctbl.c.url == rurl) \
					.delete(synchronize_session=False)

				bad += 1
				deleted += 1
			total_rows += 1

			if bad > 5000:
				bad = 0
				last_commit = deleted
				sess2.commit()
			else:
				msg = "Deleted: %s, since commit: %s, last_bad: '%s' (%s, %s%%)" % \
					(deleted, deleted-last_commit, last_bad, changed_rows, 100.0*(deleted / total_rows))
				maxlen = max(len(msg), maxlen)
				pbar.set_description(msg.ljust(maxlen), refresh=False)

			pbar.update(n=1)

	sess1.commit()
	sess2.commit()

def to_locpath(fqpath):
	assert fqpath.startswith(C_RAW_RESOURCE_DIR)

	locpath = fqpath[len(C_RAW_RESOURCE_DIR):]
	if locpath.startswith("/"):
		locpath = locpath[1:]
	return locpath

def exposed_reset_raw_in_progress():
	'''
	Reset raw downloads that are in progress.
	'''
	RawArchiver.RawUrlUpserter.resetRawInProgress()

def set_new_to_skipped():
	print("Resetting any stalled downloads from the previous session.")

	commit_interval =  50000
	step            =  50000
	commit_every    =  30
	last_commit     = time.time()

	with db.session_context(override_timeout_ms=60 * 1000 * 45) as sess:
		try:
			# sess.execute('''SET enable_bitmapscan TO off;''')
			print("Getting minimum row in need or update..")
			start = sess.execute("""SELECT min(id) FROM raw_web_pages WHERE state = 'fetching' OR state = 'processing' OR state = 'new'""")
			# start = sess.execute("""SELECT min(id) FROM raw_web_pages WHERE state = 'fetching' OR state = 'processing' OR state = 'new' OR state = 'specialty_deferred' OR state = 'specialty_ready'""")
			start = list(start)[0][0]
			if start is None:
				print("No rows to reset!")
				return
			print("Minimum row ID:", start, "getting maximum row...")
			stop = sess.execute("""SELECT max(id) FROM raw_web_pages WHERE state = 'fetching' OR state = 'processing' OR state = 'new'""")
			# stop = sess.execute("""SELECT max(id) FROM raw_web_pages WHERE state = 'fetching' OR state = 'processing' OR state = 'new' OR state = 'specialty_deferred' OR state = 'specialty_ready'""")
			stop = list(stop)[0][0]
			print("Maximum row ID: ", stop)

			print("Need to fix rows from %s to %s" % (start, stop))
			start = start - (start % step)

			changed = 0
			tot_changed = 0
			for idx in tqdm.tqdm(range(start, stop, step), desc="Resetting raw URLs"):
				try:
					# SQL String munging! I'm a bad person!
					# Only done because I can't easily find how to make sqlalchemy
					# bind parameters ignore the postgres specific cast
					# The id range forces the query planner to use a much smarter approach which is much more performant for small numbers of updates
					have = sess.execute("""UPDATE
												raw_web_pages
											SET
												state = 'skipped'
											WHERE
												(state = 'fetching' OR state = 'processing' OR state = 'new')
											AND
												id > {}
											AND
												id <= {};""".format(idx, idx+step))

					# processed  = idx - start
					# total_todo = stop - start
					changed += have.rowcount
					tot_changed += have.rowcount
					if changed > commit_interval:
						print("Committing (%s changed rows)...." % changed, end=' ')
						sess.commit()
						print("done")
						changed = 0
						last_commit     = time.time()

					if time.time() > last_commit + commit_every:
						last_commit     = time.time()
						print("Committing (%s changed rows, timed out)...." % changed, end=' ')
						sess.commit()
						print("done")
						changed = 0

				except sqlalchemy.exc.OperationalError:
					sess.rollback()
				except sqlalchemy.exc.InvalidRequestError:
					sess.rollback()

			sess.commit()

		finally:
			pass
			# sess.execute('''SET enable_bitmapscan TO on;''')

def reset_root_skipped_to_new():
	print("Initializing all start URLs in the database")
	with common.database.session_context() as sess:
		for module in RawArchiver.RawActiveModules.ACTIVE_MODULES:
			for starturl in module.get_start_urls():
				have = sess.query(common.database.RawWebPages) \
					.filter(common.database.RawWebPages.url == starturl)   \
					.scalar()
				if have:
					if have.state == 'skipped':
						have.state = 'new'
				else:
					netloc = urlFuncs.getNetLoc(starturl)
					new = common.database.RawWebPages(
							url               = starturl,
							starturl          = starturl,
							netloc            = netloc,
							priority          = common.database.DB_IDLE_PRIORITY,
							distance          = common.database.DB_DEFAULT_DIST,
						)
					print("Missing start-url for address: '{}'".format(starturl))
					sess.add(new)
				try:
					sess.commit()
				except Exception:
					print("Failure inserting start url for address: '{}'".format(starturl))

					sess.rollback()

def exposed_reset_raw_incremental_fetch():
	'''
	Reset the raw fetch incremental fetch.
	'''
	set_new_to_skipped()
	reset_root_skipped_to_new()

def exposed_reset_raw_missing():
	'''
	Retrigger all raw-archive links that don't seem to have
	a corresponding file on-disk.
	'''

	sess = db.get_db_session()

	bad = 0
	for row in sess.query(db.RawWebPages).yield_per(1000):
		if row.fspath:

			nl, rest = row.fspath.split("/", 1)
			nl = nl.split(".")
			nl.reverse()
			nl = "/".join(nl)
			newp = nl + "/" + rest

			old = os.path.join(C_RAW_RESOURCE_DIR, "old", row.fspath)
			new = os.path.join(C_RAW_RESOURCE_DIR, newp)

			if os.path.exists(new) and row.fspath == newp:
				#print("Nothing to do: ", row.fspath, new, newp)
				pass
			elif os.path.exists(new):
				print("Relinking: ", newp, row.fspath)
				row.fspath = to_locpath(new)
				bad += 1
			elif os.path.exists(old):
				dirPath = os.path.split(new)[0]
				if not os.path.exists(dirPath):
					os.makedirs(dirPath)
				shutil.move(old, new)

				row.fspath = to_locpath(new)
				bad += 1
				print("Moving: ", old, new)
			else:
				row.state = "new"
				bad += 1
		else:
			row.state = "new"
			bad += 1

		if bad > 25000:
			print("Committing!")
			bad = 0
			sess.commit()
	sess.commit()

def exposed_delete_unattached_raw_files():
	'''
	Load the local content files from the raw archiver, and compare them
	against the database. Extra files that are not in the database will then
	be deleted.
	This also resets the dlstate of for addresses where the file is missing.
	'''

def exposed_drop_raw_priorities():
	'''
	Reset the priority of every row in the table to the IDLE_PRIORITY level
	'''

	# We have a maximum commit interval so we don't hold a transaction open for extremely long periods of time,
	# as doing so can cause other portions of the system to time out.
	commit_interval_s  = 30
	step               = 10000

	with db.session_context(override_timeout_ms=30*60*1000) as sess:
		print("Getting minimum row in need or update..")
		start = sess.execute("""SELECT min(id) FROM raw_web_pages WHERE priority != 9""")
		start = list(start)[0][0]
		if start is None:
			print("No rows to reset!")
			return
		print("Minimum row ID: ", start, "getting maximum row...")
		stop = sess.execute("""SELECT max(id) FROM raw_web_pages WHERE priority != 9""")
		stop = list(stop)[0][0]
		print("Maximum row ID: ", stop)

		if not start:
			print("No null rows to fix!")
			return

		print("Need to fix rows from %s to %s" % (start, stop))
		start = start - (start % step)

		changed = 0
		changed_tot = 0
		last_commit = time.time()
		pb = tqdm.tqdm(range(start, stop, step), desc='Dropping raw priorities.')
		for idx in pb:
			try:
				# SQL String munging! I'm a bad person!
				# Only done because I can't easily find how to make sqlalchemy
				# bind parameters ignore the postgres specific cast
				# The id range forces the query planner to use a much smarter approach which is much more performant for small numbers of updates
				have = sess.execute("""update raw_web_pages set priority = 9 where priority != 9 AND id > {} AND id <= {};""".format(idx, idx+step))

				# processed  = idx - start
				# total_todo = stop - start
				desc = '(drop_priorities) -> %6i, %6i, %6i' % (have.rowcount, changed, changed_tot)
				pb.set_description(desc)

				changed += have.rowcount
				changed_tot += have.rowcount
				if changed > step * 10 or (time.time() - last_commit) > commit_interval_s:
					print("Committing (%s changed rows)...." % changed, end=' ')
					sess.commit()
					print("done")
					changed = 0
					last_commit = time.time()

			except sqlalchemy.exc.OperationalError:
				sess.rollback()
			except sqlalchemy.exc.InvalidRequestError:
				sess.rollback()

		sess.commit()


    if backend is None:
        backend = get_mail_backend()
    return backend not in settings.SENTRY_SMTP_DISABLED_BACKENDS

import subprocess
import tempfile

from django.conf import settings
from django.core.mail.backends.base import BaseEmailBackend

from sentry import options

def is_smtp_enabled(backend=None):
    """Check if the current backend is SMTP based."""

def get_mail_backend():
    backend = options.get("mail.backend")
    try:
        return settings.SENTRY_EMAIL_BACKEND_ALIASES[backend]
    except KeyError:
        return backend

class PreviewBackend(BaseEmailBackend):
    """
    Email backend that can be used in local development to open messages in the
    local mail client as they are sent.

    Probably only works on OS X.
    """

    def send_messages(self, email_messages):
        for message in email_messages:
            content = bytes(message.message())
            preview = tempfile.NamedTemporaryFile(
                delete=False, prefix="sentry-email-preview-", suffix=".eml"
            )
            try:
                preview.write(content)
                preview.flush()
            finally:
                preview.close()

            subprocess.check_call(("open", preview.name))

        return len(email_messages)


    source_file = env.get('file', None)
    if source_file:
        return f'. {__escape(source_file, env)}'
    execute = env.get('execute', None)
    if execute:
        return execute
    name = env['name']
    value = __escape(env['value'], env)
    return f'{name}={value}; export {name}'

RAW_VALUE_BY_DEFAULT = False

def env_to_statement(env):
    ''' Return the abstraction description of an environment variable definition
    into a statement for shell script.

    >>> env_to_statement(dict(name='X', value='Y'))
    'X="Y"; export X'
    >>> env_to_statement(dict(name='X', value='Y', raw=True))
    'X=Y; export X'
    >>> env_to_statement(dict(name='X', value='"A","B","C"'))
    'X="\\\\"A\\\\",\\\\"B\\\\",\\\\"C\\\\""; export X'
    >>> env_to_statement(dict(file="Y"))
    '. "Y"'
    >>> env_to_statement(dict(file="'RAW $FILE'", raw=True))
    ". 'RAW $FILE'"
    >>> # Source file takes precedence
    >>> env_to_statement(dict(name='X', value='"A","B","C"', file="S"))
    '. "S"'
    >>> env_to_statement(dict(execute="module load java/1.5.1"))
    'module load java/1.5.1'
    '''

def __escape(value, env):
    raw = env.get('raw', RAW_VALUE_BY_DEFAULT)
    if not raw:
        value = '"' + value.replace('"', '\\"') + '"'
    return value