    def func( boo ):
        pass

    assert func.__code__.co_filename == __file__
    filename			= "something/else.py"
    change_function( func, co_filename=filename )
    assert func.__code__.co_filename == filename

from __future__ import absolute_import, print_function, division
try:
    from future_builtins import zip, map # Use Python 3 "lazy" zip, map
except ImportError:
    pass

import threading

from .misc import (
    near, scale, magnitude, centeraxis, natural, change_function, mutexmethod,
    parse_ip_port, ip, network,
    hexdump, hexload,
)

def test_scale():
    assert near( scale(   0., ( 0., 100. ), ( 32., 212. )),  32. )
    assert near( scale( -40., ( 0., 100. ), ( 32., 212. )), -40. )
    assert near( scale(  20., ( 0., 100. ), ( 32., 212. )),  68. )

    # Try an inverted mapping (a reverse-ordered range)
    assert near( scale(   0., ( 0., 100. ), ( 1., -1.   )),   1. )
    assert near( scale( -40., ( 0., 100. ), ( 1., -1.   )),   1.80 )
    assert near( scale(  20., ( 0., 100. ), ( 1., -1.   )),   0.60 )
    assert near( scale( 120., ( 0., 100. ), ( 1., -1.   )),  -1.40 )

    # Try a reverse-ordered domain
    assert near( scale(   0., ( 100., 0. ), ( 32., 212. )), 212. )
    assert near( scale( -40., ( 100., 0. ), ( 32., 212. )), 284. )
    assert near( scale(  20., ( 100., 0. ), ( 32., 212. )), 176. )

    # An exponential mapping
    assert near( scale(  40,       ( 25  , 40 ), ( 0, 1 )),              1 )
    assert near( scale(  40,       ( 25  , 40 ), ( 0, 1 ), exponent=2),  1 )
    assert near( scale(  25,       ( 25  , 40 ), ( 0, 1 )),              0 )
    assert near( scale(  25,       ( 25  , 40 ), ( 0, 1 ), exponent=2),  0 )
    assert near( scale(  25+15/2 , ( 25  , 40 ), ( 0, 1 )),               .5 )
    assert near( scale(  25+15/2 , ( 25  , 40 ), ( 0, 1 ), exponent=2),   .25 )
    assert near( scale(  39      , ( 25  , 40 ), ( 0, 1 )),               .9333 )
    assert near( scale(  39      , ( 25  , 40 ), ( 0, 1 ), exponent=2),   .8711 )
    assert near( scale(  26      , ( 25  , 40 ), ( 0, 1 )),               .066667 )
    assert near( scale(  26      , ( 25  , 40 ), ( 0, 1 ), exponent=2),   .004444 )

    # Ensure non-linear scaling ensures negatives may be handled by clamping domain
    assert near( scale(  24      , ( 25  , 40 ), ( 0, 1 ), exponent=2, clamped=True ),  0 )

def test_magnitude():
    assert near( magnitude( 23.   ),  1.   )
    assert near( magnitude(   .23 ),  .01  )

    assert near( magnitude( 75.   ), 10.   )
    assert near( magnitude(   .03 ),  .001 )

    assert near( magnitude( 33., 2 ),  16. )
    assert near( magnitude( 50., 2 ),  32. )

def test_centeraxis():
    string='abc.123.xyz'
    assert centeraxis( string, 20 ) \
        == '   abc.123.xyz      '
    assert centeraxis( string, 20, reverse=True ) \
        == '       abc.123.xyz  '
    assert centeraxis( string, 20, reverse=True, fillchar='x' ) \
        == 'xxxxxxxabc.123.xyzxx'

    string='----------abc.123.xyz++++++++++'
    assert centeraxis( string, 20, reverse=True ) \
        == '----------abc.123.xyz++++++++++'
    assert centeraxis( string, 20, reverse=True, clip=True ) \
        == '-------abc.123.xyz++'
    assert centeraxis( string, 20,               clip=True ) \
        == '---abc.123.xyz++++++'
    assert centeraxis( string, 20,               clip=True ) \
        == '---abc.123.xyz++++++'

    # If no axis char, center around a fill char
    string='abc'
    assert centeraxis( string, 20, reverse=True ) \
        == '       abc          '
    assert centeraxis( string, 20               ) \
        == '           abc      '

def test_natural():
    assert natural('10th')	== ('       10', 't', 'h')
    for itm in [ None, [1], {'a':1} ]:
        assert natural(itm)	== ('', itm.__class__.__name__,
                                    "%9d" % ( hash( itm )
                                              if hasattr( itm, '__hash__' ) and itm.__hash__ is not None
                                              else id( itm )))

    l = ['10th', '1st', '9', 9, 'Z', 'a', 9.0, "9.1", None ]
    ls1 = ['None', "'1st'", "'9'", "9", "9.0", "'9.1'", "'10th'", "'a'", "'Z'"]
    ls2 = ['None', "'1st'", "9", "'9'", "9.0", "'9.1'", "'10th'", "'a'", "'Z'"]

    s = sorted( l, key=natural )
    rs = [ repr(i) for i in s ]
    assert rs == ls1 or rs == ls2

def test_function_creation():
    """Creating functions with code containing a defined co_filename is required in
    order to extend the logging module.  Unfortunately, this module seeks up the
    stack frame until it finds a function whose co_filename is not the logging
    module...  """

def test_mutexmethod():

    class C( object ):
        _cls_lock		= threading.Lock()
        def __init__( self ):
            self._ins_lock	= threading.Lock()

        @classmethod
        @mutexmethod( '_cls_lock', blocking=False )
        def clsmethod_lock_cls( cls, f=None ):
            if f:
                return f()

        @mutexmethod( '_cls_lock', blocking=False )
        def insmethod_lock_cls( self, f=None ):
            if f:
                return f()

        @mutexmethod( '_ins_lock', blocking=False )
        def insmethod_lock_ins( self, f=None ):
            if f:
                return f()

    c				= C()

    # Same lock; should raise Exception (since blocking=False used above)
    assert c.insmethod_lock_cls() is None
    try:
        c.insmethod_lock_cls( c.insmethod_lock_cls )
        assert False, "Should have raised recursive lock exception"
    except Exception as exc:
        assert "Lock is held" in str( exc )

    assert c.clsmethod_lock_cls() is None
    try:
        c.clsmethod_lock_cls( c.clsmethod_lock_cls )
        assert False, "Should have raised recursive lock exception"
    except Exception as exc:
        assert "Lock is held" in str( exc )

    try:
        c.clsmethod_lock_cls( c.insmethod_lock_cls )
        assert False, "Should have raised recursive lock exception"
    except Exception as exc:
        assert "Lock is held" in str( exc )

    # Two different locks; should not interfere
    c.clsmethod_lock_cls( c.insmethod_lock_ins )

def test_parse_ip_port():

    for t,(a,p) in {
        "127.0.0.1":			( ip("127.0.0.1"),			None ),
        "127.0.0.1:80":			( ip("127.0.0.1"),			80 ),
        "::1":				( ip("::1"),				None ),
        "[::1]:80":			( ip("::1"),				80 ),
        "::192.168.0.1":		( ip("::c0a8:1"), 			None ),
        "2605:2700:0:3::4713:93e3":	( ip("2605:2700:0:3::4713:93e3"),	None ),
        "[2605:2700:0:3::4713:93e3]:80":( ip("2605:2700:0:3::4713:93e3"),	80),
        "boogaloo.cash:443":		( "boogaloo.cash",			443 ),
    }.items():
        assert parse_ip_port( t ) == (a,p)

    for t,(a,p) in {
        "127.0.0.1":			( ip("127.0.0.1"),			123 ),
        "127.0.0.1:80":			( ip("127.0.0.1"),			80 ),
        "::1":				( ip("::1"),				123 ),
        "[::1]:80":			( ip("::1"),				80 ),
        "::192.168.0.1":		( ip("::c0a8:1"), 			123 ),
        "2605:2700:0:3::4713:93e3":	( ip("2605:2700:0:3::4713:93e3"),	123 ),
        "[2605:2700:0:3::4713:93e3]:80":( ip("2605:2700:0:3::4713:93e3"),	80),
        "boogaloo.cash:443":		( "boogaloo.cash",			443 ),
    }.items():
        assert parse_ip_port( t, default=(None,123) ) == (a,p)

    assert parse_ip_port( ":11", default=('',456)) == ('',11)
    assert parse_ip_port( ":11", default=('boo',456)) == ('boo',11)
    assert parse_ip_port( "", default=('',456)) == ('',456)

    assert str( network( "192.168.1.0/24" ).broadcast_address ) == "192.168.1.255"

def test_hexdump():
    vals		= b'\x01\x02\x03'

    dump		= hexdump( vals )
    assert dump == '00000000:  01 02 03                                           |...|'
    load		= hexload( dump )
    print( repr( load ))
    assert load == vals

    dump		= hexdump( vals, quote='' )
    assert dump == '00000000:  01 02 03                                           ...'
    load		= hexload( dump )
    print( repr( load ))
    assert load == vals

    dump		= hexdump( vals, quote='' )
    assert dump == '00000000:  01 02 03                                           ...'
    load		= hexload( dump, offset=1 )
    print( repr( load ))
    assert load == vals[1:]

    # fill gaps in addresses, start at an offset
    gaps		= r"""
        00003FD0:  3F D0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |................|
        ... something unparsable ...
        00003FF0:  3F F0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |................|
        00004000:  40 00 30 31 20 53 45 34  20 45 20 32 33 2e 35 63   |@.01 SE4 E 23.5c|
    """
    load		= hexload( gaps, offset=0x3fc0, fill=b'\xFF', skip=True )
    dump		= hexdump( load, offset=0x3fc0 )
    print( dump )
    assert dump == """\
00003FC0:  ff ff ff ff ff ff ff ff  ff ff ff ff ff ff ff ff   |................|
00003FD0:  3f d0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |?...............|
00003FE0:  ff ff ff ff ff ff ff ff  ff ff ff ff ff ff ff ff   |................|
00003FF0:  3f f0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |?...............|
00004000:  40 00 30 31 20 53 45 34  20 45 20 32 33 2e 35 63   |@.01 SE4 E 23.5c|"""

    # Don't fill, skip unparsable rows
    load		= hexload( gaps, skip=True )
    dump		= hexdump( load )
    print( dump )
    assert dump == """\
00000000:  3f d0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |?...............|
00000010:  3f f0 00 00 00 00 00 00  00 00 00 00 12 00 00 00   |?...............|
00000020:  40 00 30 31 20 53 45 34  20 45 20 32 33 2e 35 63   |@.01 SE4 E 23.5c|"""
    try:
        load		= hexload( gaps )
    except Exception as exc:
        assert str(exc) == "Failed to match a hex dump on row: '        ... something unparsable ...'"
    else:
        raise AssertionError( "Failed to raise Exception, got: {load!r}".format( load=load ))

    # simpler formats that are accepted
    simp		= """\
        0001:01020304
        0005:05
        9:09
    """
    load		= hexload( simp )
    print( repr( load ) )
    dump		= hexdump( load )
    print( dump )
    assert dump == """\
00000000:  01 02 03 04 05 09                                  |......|"""

    load		= hexload( simp, fill=b'\x99' )
    print( repr( load ) )
    dump		= hexdump( load )
    print( dump )
    assert dump == """\
00000000:  99 01 02 03 04 05 99 99  99 09                     |..........|"""


while True:
        filePath = os.path.join(_, ".git", "HEAD")
        if os.path.exists(filePath):
            break
        else:
            filePath = None
            if _ == os.path.dirname(_):
                break
            else:
                _ = os.path.dirname(_)

"""
"""

import os
import re
import subprocess

def getRevisionNumber():
    """
    Returns abbreviated commit hash number as retrieved with "git rev-parse --short HEAD"
    """

    retVal = None
    filePath = None
    _ = os.path.dirname(__file__)

    while True:
        if filePath and os.path.isfile(filePath):
            with open(filePath, "r") as f:
                content = f.read()
                filePath = None
                if content.startswith("ref: "):
                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
                else:
                    match = re.match(r"(?i)[0-9a-f]{32}", content)
                    retVal = match.group(0) if match else None
                    break
        else:
            break

    if not retVal:
        process = subprocess.Popen("git rev-parse --verify HEAD", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, _ = process.communicate()
        match = re.search(r"(?i)[0-9a-f]{32}", stdout or "")
        retVal = match.group(0) if match else None

    return retVal[:7] if retVal else None


from __clrclasses__.Autodesk.AutoCAD.DatabaseServices import ObjectId as _n_0_t_0
from __clrclasses__.Autodesk.AutoCAD.DatabaseServices import LayerTableRecord as _n_0_t_1
from __clrclasses__.Autodesk.AutoCAD.Runtime import RXObject as _n_1_t_0
from __clrclasses__.System import Array as _n_2_t_0
from __clrclasses__.System import IDisposable as _n_2_t_1
from __clrclasses__.System import ICloneable as _n_2_t_2
from __clrclasses__.System import Enum as _n_2_t_3
from __clrclasses__.System import IComparable as _n_2_t_4
from __clrclasses__.System import IFormattable as _n_2_t_5
from __clrclasses__.System import IConvertible as _n_2_t_6
from __clrclasses__.System import ValueType as _n_2_t_7
from __clrclasses__.System import IntPtr as _n_2_t_8
from __clrclasses__.System.Collections import ICollection as _n_3_t_0
import typing
class AndExpression(object):
    def GetRelationalExpressions(self) -> _n_2_t_0[RelationalExpression]:...
class LayerCollection(_n_3_t_0, typing.Iterable[_n_0_t_0]):
    @property
    def Item(self) -> _n_0_t_0:"""Item { get; } -> ObjectId"""
    def Add(self, value: _n_0_t_0):...
    def Clear(self):...
    def Contains(self, value: _n_0_t_0) -> bool:...
    def Remove(self, value: _n_0_t_0):...
class LayerFilter(_n_1_t_0, _n_2_t_1, _n_2_t_2):
    @property
    def AllowDelete(self) -> bool:"""AllowDelete { get; } -> bool"""
    @property
    def AllowNested(self) -> bool:"""AllowNested { get; } -> bool"""

    @property
    def AllowRename(self) -> bool:"""AllowRename { get; } -> bool"""
    @property
    def DisplayImages(self) -> LayerFilterDisplayImages:"""DisplayImages { get; } -> LayerFilterDisplayImages"""
    @property
    def DynamicallyGenerated(self) -> bool:"""DynamicallyGenerated { get; } -> bool"""
    @property
    def FilterExpression(self) -> str:"""FilterExpression { get; set; } -> str"""
    @property
    def IsIdFilter(self) -> bool:"""IsIdFilter { get; } -> bool"""
    @property
    def IsProxy(self) -> bool:"""IsProxy { get; } -> bool"""
    @property
    def Name(self) -> str:"""Name { get; set; } -> str"""
    @property
    def NestedFilters(self) -> LayerFilterCollection:"""NestedFilters { get; } -> LayerFilterCollection"""
    @property
    def Parent(self) -> LayerFilter:"""Parent { get; } -> LayerFilter"""
    def __init__(self) -> LayerFilter:...
    def Filter(self, layer: _n_0_t_1) -> bool:...
    def GenerateNested(self):...
    def GetFilterExpressionTree(self) -> _n_2_t_0[AndExpression]:...
    def ShowEditor(self) -> LayerFilter.DialogResult:...
    class DialogResult(_n_2_t_3, _n_2_t_4, _n_2_t_5, _n_2_t_6):
        Cancel: int
        OK: int
        UseDefault: int
        value__: int
class LayerFilterCollection(_n_3_t_0, typing.Iterable[LayerFilter]):
    @property
    def Item(self) -> LayerFilter:"""Item { get; } -> LayerFilter"""
    def Add(self, value: LayerFilter):...
    def Clear(self):...
    def Contains(self, value: LayerFilter) -> bool:...
    def Remove(self, value: LayerFilter):...
class LayerFilterDisplayImages(_n_2_t_7):
    @property
    def ImageIndex(self) -> int:"""ImageIndex { get; } -> int"""
    @property
    def ImageListHandle(self) -> _n_2_t_8:"""ImageListHandle { get; } -> IntPtr"""
    @property
    def SelectedImageIndex(self) -> int:"""SelectedImageIndex { get; } -> int"""
    def __init__(self, imageListHandle: _n_2_t_8, imageIndex: int, selectedImageIndex: int) -> LayerFilterDisplayImages:...
class LayerFilterTree(_n_2_t_7):
    @property
    def Current(self) -> LayerFilter:"""Current { get; } -> LayerFilter"""
    @property
    def Root(self) -> LayerFilter:"""Root { get; } -> LayerFilter"""
    def __init__(self, root: LayerFilter, current: LayerFilter) -> LayerFilterTree:...
class LayerGroup(LayerFilter, _n_2_t_1, _n_2_t_2):
    @property
    def LayerIds(self) -> LayerCollection:"""LayerIds { get; } -> LayerCollection"""
    def __init__(self) -> LayerGroup:...
class RelationalExpression(object):
    @property
    def Constant(self) -> str:"""Constant { get; } -> str"""
    @property
    def Variable(self) -> str:"""Variable { get; } -> str"""


    lines = header.split('\n')
    sep = '\n' + 8 * ' '
    return sep.join(lines)

"""distutils.util

Miscellaneous utility functions -- anything that doesn't fit into
one of the other *util.py modules.
"""

import os
import re
import importlib.util
import string
import sys
from distutils.errors import DistutilsPlatformError
from distutils.dep_util import newer
from distutils.spawn import spawn
from distutils import log
from distutils.errors import DistutilsByteCompileError

def get_platform ():
    """Return a string that identifies the current platform.  This is used mainly to
    distinguish platform-specific build directories and platform-specific built
    distributions.  Typically includes the OS name and version and the
    architecture (as supplied by 'os.uname()'), although the exact information
    included depends on the OS; eg. on Linux, the kernel version isn't
    particularly important.

    Examples of returned values:
       linux-i586
       linux-alpha (?)
       solaris-2.6-sun4u

    Windows will return one of:
       win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
       win32 (all others - specifically, sys.platform is returned)

    For other non-POSIX platforms, currently just returns 'sys.platform'.

    """
    if os.name == 'nt':
        if 'amd64' in sys.version.lower():
            return 'win-amd64'
        return sys.platform

    # Set for cross builds explicitly
    if "_PYTHON_HOST_PLATFORM" in os.environ:
        return os.environ["_PYTHON_HOST_PLATFORM"]

    if os.name != "posix" or not hasattr(os, 'uname'):
        # XXX what about the architecture? NT is Intel or Alpha,
        # Mac OS is M68k or PPC, etc.
        return sys.platform

    # Try to distinguish various flavours of Unix

    (osname, host, release, version, machine) = os.uname()

    # Convert the OS name to lowercase, remove '/' characters, and translate
    osname = osname.lower().replace('/', '')
    machine = machine.replace(' ', '_')
    machine = machine.replace('/', '-')

    if osname[:5] == "linux":
        # At least on Linux/Intel, 'machine' is the processor --
        # XXX what about Alpha, SPARC, etc?
        return  "%s-%s" % (osname, machine)
    elif osname[:5] == "sunos":
        if release[0] >= "5":           # SunOS 5 == Solaris 2
            osname = "solaris"
            release = "%d.%s" % (int(release[0]) - 3, release[2:])
            # bootstrap problem. We use a dict to get an error
            # if some suspicious happens.
            bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
            machine += ".%s" % bitness[sys.maxsize]
        # fall through to standard osname-release-machine representation
    elif osname[:3] == "aix":
        return "%s-%s.%s" % (osname, version, release)
    elif osname[:6] == "cygwin":
        osname = "cygwin"
        rel_re = re.compile (r'[\d.]+', re.ASCII)
        m = rel_re.match(release)
        if m:
            release = m.group()
    elif osname[:6] == "darwin":
        import _osx_support, distutils.sysconfig
        osname, release, machine = _osx_support.get_platform_osx(
                                        distutils.sysconfig.get_config_vars(),
                                        osname, release, machine)

    return "%s-%s-%s" % (osname, release, machine)

def convert_path (pathname):
    """Return 'pathname' as a name that will work on the native filesystem,
    i.e. split it on '/' and put it back together again using the current
    directory separator.  Needed because filenames in the setup script are
    always supplied in Unix style, and have to be converted to the local
    convention before we can actually use them in the filesystem.  Raises
    ValueError on non-Unix-ish systems if 'pathname' either starts or
    ends with a slash.
    """
    if os.sep == '/':
        return pathname
    if not pathname:
        return pathname
    if pathname[0] == '/':
        raise ValueError("path '%s' cannot be absolute" % pathname)
    if pathname[-1] == '/':
        raise ValueError("path '%s' cannot end with '/'" % pathname)

    paths = pathname.split('/')
    while '.' in paths:
        paths.remove('.')
    if not paths:
        return os.curdir
    return os.path.join(*paths)

def change_root (new_root, pathname):
    """Return 'pathname' with 'new_root' prepended.  If 'pathname' is
    relative, this is equivalent to "os.path.join(new_root,pathname)".
    Otherwise, it requires making 'pathname' relative and then joining the
    two, which is tricky on DOS/Windows and Mac OS.
    """
    if os.name == 'posix':
        if not os.path.isabs(pathname):
            return os.path.join(new_root, pathname)
        else:
            return os.path.join(new_root, pathname[1:])

    elif os.name == 'nt':
        (drive, path) = os.path.splitdrive(pathname)
        if path[0] == '\\':
            path = path[1:]
        return os.path.join(new_root, path)

    else:
        raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)

_environ_checked = 0
def check_environ ():
    """Ensure that 'os.environ' has all the environment variables we
    guarantee that users can use in config files, command-line options,
    etc.  Currently this includes:
      HOME - user's home directory (Unix only)
      PLAT - description of the current platform, including hardware
             and OS (see 'get_platform()')
    """
    global _environ_checked
    if _environ_checked:
        return

    if os.name == 'posix' and 'HOME' not in os.environ:
        import pwd
        os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]

    if 'PLAT' not in os.environ:
        os.environ['PLAT'] = get_platform()

    _environ_checked = 1

def subst_vars (s, local_vars):
    """Perform shell/Perl-style variable substitution on 'string'.  Every
    occurrence of '$' followed by a name is considered a variable, and
    variable is substituted by the value found in the 'local_vars'
    dictionary, or in 'os.environ' if it's not in 'local_vars'.
    'os.environ' is first checked/augmented to guarantee that it contains
    certain values: see 'check_environ()'.  Raise ValueError for any
    variables not found in either 'local_vars' or 'os.environ'.
    """
    check_environ()
    def _subst (match, local_vars=local_vars):
        var_name = match.group(1)
        if var_name in local_vars:
            return str(local_vars[var_name])
        else:
            return os.environ[var_name]

    try:
        return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
    except KeyError as var:
        raise ValueError("invalid variable '$%s'" % var)

def grok_environment_error (exc, prefix="error: "):
    # Function kept for backward compatibility.
    # Used to try clever things with EnvironmentErrors,
    # but nowadays str(exception) produces good messages.
    return prefix + str(exc)

_wordchars_re = _squote_re = _dquote_re = None
def _init_regex():
    global _wordchars_re, _squote_re, _dquote_re
    _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
    _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
    _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')

def split_quoted (s):
    """Split a string up according to Unix shell-like rules for quotes and
    backslashes.  In short: words are delimited by spaces, as long as those
    spaces are not escaped by a backslash, or inside a quoted string.
    Single and double quotes are equivalent, and the quote characters can
    be backslash-escaped.  The backslash is stripped from any two-character
    escape sequence, leaving only the escaped character.  The quote
    characters are stripped from any quoted string.  Returns a list of
    words.
    """

    # This is a nice algorithm for splitting up a single string, since it
    # doesn't require character-by-character examination.  It was a little
    # bit of a brain-bender to get it working right, though...
    if _wordchars_re is None: _init_regex()

    s = s.strip()
    words = []
    pos = 0

    while s:
        m = _wordchars_re.match(s, pos)
        end = m.end()
        if end == len(s):
            words.append(s[:end])
            break

        if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
            words.append(s[:end])       # we definitely have a word delimiter
            s = s[end:].lstrip()
            pos = 0

        elif s[end] == '\\':            # preserve whatever is being escaped;
                                        # will become part of the current word
            s = s[:end] + s[end+1:]
            pos = end+1

        else:
            if s[end] == "'":           # slurp singly-quoted string
                m = _squote_re.match(s, end)
            elif s[end] == '"':         # slurp doubly-quoted string
                m = _dquote_re.match(s, end)
            else:
                raise RuntimeError("this can't happen (bad char '%c')" % s[end])

            if m is None:
                raise ValueError("bad string (mismatched %s quotes?)" % s[end])

            (beg, end) = m.span()
            s = s[:beg] + s[beg+1:end-1] + s[end:]
            pos = m.end() - 2

        if pos >= len(s):
            words.append(s)
            break

    return words

def execute (func, args, msg=None, verbose=0, dry_run=0):
    """Perform some action that affects the outside world (eg.  by
    writing to the filesystem).  Such actions are special because they
    are disabled by the 'dry_run' flag.  This method takes care of all
    that bureaucracy for you; all you have to do is supply the
    function to call and an argument tuple for it (to embody the
    "external action" being performed), and an optional message to
    print.
    """
    if msg is None:
        msg = "%s%r" % (func.__name__, args)
        if msg[-2:] == ',)':        # correct for singleton tuple
            msg = msg[0:-2] + ')'

    log.info(msg)
    if not dry_run:
        func(*args)

def strtobool (val):
    """Convert a string representation of truth to true (1) or false (0).

    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
    'val' is anything else.
    """
    val = val.lower()
    if val in ('y', 'yes', 't', 'true', 'on', '1'):
        return 1
    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
        return 0
    else:
        raise ValueError("invalid truth value %r" % (val,))

def byte_compile (py_files,
                  optimize=0, force=0,
                  prefix=None, base_dir=None,
                  verbose=1, dry_run=0,
                  direct=None):
    """Byte-compile a collection of Python source files to .pyc
    files in a __pycache__ subdirectory.  'py_files' is a list
    of files to compile; any files that don't end in ".py" are silently
    skipped.  'optimize' must be one of the following:
      0 - don't optimize
      1 - normal optimization (like "python -O")
      2 - extra optimization (like "python -OO")
    If 'force' is true, all files are recompiled regardless of
    timestamps.

    The source filename encoded in each bytecode file defaults to the
    filenames listed in 'py_files'; you can modify these with 'prefix' and
    'basedir'.  'prefix' is a string that will be stripped off of each
    source filename, and 'base_dir' is a directory name that will be
    prepended (after 'prefix' is stripped).  You can supply either or both
    (or neither) of 'prefix' and 'base_dir', as you wish.

    If 'dry_run' is true, doesn't actually do anything that would
    affect the filesystem.

    Byte-compilation is either done directly in this interpreter process
    with the standard py_compile module, or indirectly by writing a
    temporary script and executing it.  Normally, you should let
    'byte_compile()' figure out to use direct compilation or not (see
    the source for details).  The 'direct' flag is used by the script
    generated in indirect mode; unless you know what you're doing, leave
    it set to None.
    """

    # Late import to fix a bootstrap issue: _posixsubprocess is built by
    # setup.py, but setup.py uses distutils.
    import subprocess

    if sys.dont_write_bytecode:
        raise DistutilsByteCompileError('byte-compiling is disabled.')

    # First, if the caller didn't force us into direct or indirect mode,
    # figure out which mode we should be in.  We take a conservative
    # approach: choose direct mode *only* if the current interpreter is
    # or -OO), we don't know which level of optimization this
    # interpreter is running with, so we can't do direct
    # byte-compilation and be certain that it's the right thing.  Thus,
    # always compile indirectly if the current interpreter is in either
    # optimize mode, or if either optimization level was requested by
    # the caller.
    if direct is None:
        direct = (__debug__ and optimize == 0)

    # "Indirect" byte-compilation: write a temporary script and then
    # run it with the appropriate flags.
    if not direct:
        try:
            from tempfile import mkstemp
            (script_fd, script_name) = mkstemp(".py")
        except ImportError:
            from tempfile import mktemp
            (script_fd, script_name) = None, mktemp(".py")
        log.info("writing byte-compilation script '%s'", script_name)
        if not dry_run:
            if script_fd is not None:
                script = os.fdopen(script_fd, "w")
            else:
                script = open(script_name, "w")

            script.write("""\
from distutils.util import byte_compile
files = [
""")

            # XXX would be nice to write absolute filenames, just for
            # safety's sake (script should be more robust in the face of
            # chdir'ing before running it).  But this requires abspath'ing
            # 'prefix' as well, and that breaks the hack in build_lib's
            # 'byte_compile()' method that carefully tacks on a trailing
            # slash (os.sep really) to make sure the prefix here is "just
            # right".  This whole prefix business is rather delicate -- the
            # problem is that it's really a directory, but I'm treating it
            # as a dumb string, so trailing slashes and so forth matter.

            script.write(",\n".join(map(repr, py_files)) + "]\n")
            script.write("""
byte_compile(files, optimize=%r, force=%r,
             prefix=%r, base_dir=%r,
             verbose=%r, dry_run=0,
             direct=1)
""" % (optimize, force, prefix, base_dir, verbose))

            script.close()

        cmd = [sys.executable]
        cmd.extend(subprocess._optim_args_from_interpreter_flags())
        cmd.append(script_name)
        spawn(cmd, dry_run=dry_run)
        execute(os.remove, (script_name,), "removing %s" % script_name,
                dry_run=dry_run)

    # "Direct" byte-compilation: use the py_compile module to compile
    # right here, right now.  Note that the script generated in indirect
    # mode simply calls 'byte_compile()' in direct mode, a weird sort of
    # cross-process recursion.  Hey, it works!
    else:
        from py_compile import compile

        for file in py_files:
            if file[-3:] != ".py":
                # This lets us be lazy and not filter filenames in
                continue

            # Terminology from the py_compile module:
            #   cfile - byte-compiled file
            #   dfile - purported source filename (same as 'file' by default)
            if optimize >= 0:
                opt = '' if optimize == 0 else optimize
                cfile = importlib.util.cache_from_source(
                    file, optimization=opt)
            else:
                cfile = importlib.util.cache_from_source(file)
            dfile = file
            if prefix:
                if file[:len(prefix)] != prefix:
                    raise ValueError("invalid prefix: filename %r doesn't start with %r"
                           % (file, prefix))
                dfile = dfile[len(prefix):]
            if base_dir:
                dfile = os.path.join(base_dir, dfile)

            cfile_base = os.path.basename(cfile)
            if direct:
                if force or newer(file, cfile):
                    log.info("byte-compiling %s to %s", file, cfile_base)
                    if not dry_run:
                        compile(file, cfile, dfile)
                else:
                    log.debug("skipping byte-compilation of %s to %s",
                              file, cfile_base)

def rfc822_escape (header):
    """Return a version of the string escaped for inclusion in an
    RFC-822 header, by ensuring there are 8 spaces space after each newline.
    """

# 2to3 support

def run_2to3(files, fixer_names=None, options=None, explicit=None):
    """Invoke 2to3 on a list of Python files.
    The files should all come from the build area, as the
    modification is done in-place. To reduce the build time,
    only files modified since the last invocation of this
    function should be passed in the files argument."""

    if not files:
        return

    # Make this class local, to delay import of 2to3
    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
    class DistutilsRefactoringTool(RefactoringTool):
        def log_error(self, msg, *args, **kw):
            log.error(msg, *args)

        def log_message(self, msg, *args):
            log.info(msg, *args)

        def log_debug(self, msg, *args):
            log.debug(msg, *args)

    if fixer_names is None:
        fixer_names = get_fixers_from_package('lib2to3.fixes')
    r = DistutilsRefactoringTool(fixer_names, options=options)
    r.refactor(files, write=True)

def copydir_run_2to3(src, dest, template=None, fixer_names=None,
                     options=None, explicit=None):
    """Recursively copy a directory, only copying new and changed files,
    running run_2to3 over all newly copied Python modules afterward.

    If you give a template string, it's parsed like a MANIFEST.in.
    """
    from distutils.dir_util import mkpath
    from distutils.file_util import copy_file
    from distutils.filelist import FileList
    filelist = FileList()
    curdir = os.getcwd()
    os.chdir(src)
    try:
        filelist.findall()
    finally:
        os.chdir(curdir)
    filelist.files[:] = filelist.allfiles
    if template:
        for line in template.splitlines():
            line = line.strip()
            if not line: continue
            filelist.process_template_line(line)
    copied = []
    for filename in filelist.files:
        outname = os.path.join(dest, filename)
        mkpath(os.path.dirname(outname))
        res = copy_file(os.path.join(src, filename), outname, update=1)
        if res[1]: copied.append(outname)
    run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
             fixer_names=fixer_names, options=options, explicit=explicit)
    return copied

class Mixin2to3:
    '''Mixin class for commands that run 2to3.
    To configure 2to3, setup scripts may either change
    the class variables, or inherit from individual commands
    to override how 2to3 is invoked.'''

    # provide list of fixers to run;
    # defaults to all from lib2to3.fixers
    fixer_names = None

    # options dictionary
    options = None

    # list of fixers to invoke even though they are marked as explicit
    explicit = None

    def run_2to3(self, files):
        return run_2to3(files, self.fixer_names, self.options, self.explicit)


while '.' in paths:
        paths.remove('.')

"""distutils.util

Miscellaneous utility functions -- anything that doesn't fit into
one of the other *util.py modules.
"""

import os
import re
import importlib.util
import string
import sys
from distutils.errors import DistutilsPlatformError
from distutils.dep_util import newer
from distutils.spawn import spawn
from distutils import log
from distutils.errors import DistutilsByteCompileError

def get_platform ():
    """Return a string that identifies the current platform.  This is used mainly to
    distinguish platform-specific build directories and platform-specific built
    distributions.  Typically includes the OS name and version and the
    architecture (as supplied by 'os.uname()'), although the exact information
    included depends on the OS; eg. on Linux, the kernel version isn't
    particularly important.

    Examples of returned values:
       linux-i586
       linux-alpha (?)
       solaris-2.6-sun4u

    Windows will return one of:
       win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
       win32 (all others - specifically, sys.platform is returned)

    For other non-POSIX platforms, currently just returns 'sys.platform'.

    """
    if os.name == 'nt':
        if 'amd64' in sys.version.lower():
            return 'win-amd64'
        return sys.platform

    # Set for cross builds explicitly
    if "_PYTHON_HOST_PLATFORM" in os.environ:
        return os.environ["_PYTHON_HOST_PLATFORM"]

    if os.name != "posix" or not hasattr(os, 'uname'):
        # XXX what about the architecture? NT is Intel or Alpha,
        # Mac OS is M68k or PPC, etc.
        return sys.platform

    # Try to distinguish various flavours of Unix

    (osname, host, release, version, machine) = os.uname()

    # Convert the OS name to lowercase, remove '/' characters, and translate
    osname = osname.lower().replace('/', '')
    machine = machine.replace(' ', '_')
    machine = machine.replace('/', '-')

    if osname[:5] == "linux":
        # At least on Linux/Intel, 'machine' is the processor --
        # XXX what about Alpha, SPARC, etc?
        return  "%s-%s" % (osname, machine)
    elif osname[:5] == "sunos":
        if release[0] >= "5":           # SunOS 5 == Solaris 2
            osname = "solaris"
            release = "%d.%s" % (int(release[0]) - 3, release[2:])
            # bootstrap problem. We use a dict to get an error
            # if some suspicious happens.
            bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
            machine += ".%s" % bitness[sys.maxsize]
        # fall through to standard osname-release-machine representation
    elif osname[:3] == "aix":
        return "%s-%s.%s" % (osname, version, release)
    elif osname[:6] == "cygwin":
        osname = "cygwin"
        rel_re = re.compile (r'[\d.]+', re.ASCII)
        m = rel_re.match(release)
        if m:
            release = m.group()
    elif osname[:6] == "darwin":
        import _osx_support, distutils.sysconfig
        osname, release, machine = _osx_support.get_platform_osx(
                                        distutils.sysconfig.get_config_vars(),
                                        osname, release, machine)

    return "%s-%s-%s" % (osname, release, machine)

def convert_path (pathname):
    """Return 'pathname' as a name that will work on the native filesystem,
    i.e. split it on '/' and put it back together again using the current
    directory separator.  Needed because filenames in the setup script are
    always supplied in Unix style, and have to be converted to the local
    convention before we can actually use them in the filesystem.  Raises
    ValueError on non-Unix-ish systems if 'pathname' either starts or
    ends with a slash.
    """
    if os.sep == '/':
        return pathname
    if not pathname:
        return pathname
    if pathname[0] == '/':
        raise ValueError("path '%s' cannot be absolute" % pathname)
    if pathname[-1] == '/':
        raise ValueError("path '%s' cannot end with '/'" % pathname)

    paths = pathname.split('/')

    if not paths:
        return os.curdir
    return os.path.join(*paths)

def change_root (new_root, pathname):
    """Return 'pathname' with 'new_root' prepended.  If 'pathname' is
    relative, this is equivalent to "os.path.join(new_root,pathname)".
    Otherwise, it requires making 'pathname' relative and then joining the
    two, which is tricky on DOS/Windows and Mac OS.
    """
    if os.name == 'posix':
        if not os.path.isabs(pathname):
            return os.path.join(new_root, pathname)
        else:
            return os.path.join(new_root, pathname[1:])

    elif os.name == 'nt':
        (drive, path) = os.path.splitdrive(pathname)
        if path[0] == '\\':
            path = path[1:]
        return os.path.join(new_root, path)

    else:
        raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)

_environ_checked = 0
def check_environ ():
    """Ensure that 'os.environ' has all the environment variables we
    guarantee that users can use in config files, command-line options,
    etc.  Currently this includes:
      HOME - user's home directory (Unix only)
      PLAT - description of the current platform, including hardware
             and OS (see 'get_platform()')
    """
    global _environ_checked
    if _environ_checked:
        return

    if os.name == 'posix' and 'HOME' not in os.environ:
        import pwd
        os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]

    if 'PLAT' not in os.environ:
        os.environ['PLAT'] = get_platform()

    _environ_checked = 1

def subst_vars (s, local_vars):
    """Perform shell/Perl-style variable substitution on 'string'.  Every
    occurrence of '$' followed by a name is considered a variable, and
    variable is substituted by the value found in the 'local_vars'
    dictionary, or in 'os.environ' if it's not in 'local_vars'.
    'os.environ' is first checked/augmented to guarantee that it contains
    certain values: see 'check_environ()'.  Raise ValueError for any
    variables not found in either 'local_vars' or 'os.environ'.
    """
    check_environ()
    def _subst (match, local_vars=local_vars):
        var_name = match.group(1)
        if var_name in local_vars:
            return str(local_vars[var_name])
        else:
            return os.environ[var_name]

    try:
        return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
    except KeyError as var:
        raise ValueError("invalid variable '$%s'" % var)

def grok_environment_error (exc, prefix="error: "):
    # Function kept for backward compatibility.
    # Used to try clever things with EnvironmentErrors,
    # but nowadays str(exception) produces good messages.
    return prefix + str(exc)

_wordchars_re = _squote_re = _dquote_re = None
def _init_regex():
    global _wordchars_re, _squote_re, _dquote_re
    _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
    _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
    _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')

def split_quoted (s):
    """Split a string up according to Unix shell-like rules for quotes and
    backslashes.  In short: words are delimited by spaces, as long as those
    spaces are not escaped by a backslash, or inside a quoted string.
    Single and double quotes are equivalent, and the quote characters can
    be backslash-escaped.  The backslash is stripped from any two-character
    escape sequence, leaving only the escaped character.  The quote
    characters are stripped from any quoted string.  Returns a list of
    words.
    """

    # This is a nice algorithm for splitting up a single string, since it
    # doesn't require character-by-character examination.  It was a little
    # bit of a brain-bender to get it working right, though...
    if _wordchars_re is None: _init_regex()

    s = s.strip()
    words = []
    pos = 0

    while s:
        m = _wordchars_re.match(s, pos)
        end = m.end()
        if end == len(s):
            words.append(s[:end])
            break

        if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
            words.append(s[:end])       # we definitely have a word delimiter
            s = s[end:].lstrip()
            pos = 0

        elif s[end] == '\\':            # preserve whatever is being escaped;
                                        # will become part of the current word
            s = s[:end] + s[end+1:]
            pos = end+1

        else:
            if s[end] == "'":           # slurp singly-quoted string
                m = _squote_re.match(s, end)
            elif s[end] == '"':         # slurp doubly-quoted string
                m = _dquote_re.match(s, end)
            else:
                raise RuntimeError("this can't happen (bad char '%c')" % s[end])

            if m is None:
                raise ValueError("bad string (mismatched %s quotes?)" % s[end])

            (beg, end) = m.span()
            s = s[:beg] + s[beg+1:end-1] + s[end:]
            pos = m.end() - 2

        if pos >= len(s):
            words.append(s)
            break

    return words

def execute (func, args, msg=None, verbose=0, dry_run=0):
    """Perform some action that affects the outside world (eg.  by
    writing to the filesystem).  Such actions are special because they
    are disabled by the 'dry_run' flag.  This method takes care of all
    that bureaucracy for you; all you have to do is supply the
    function to call and an argument tuple for it (to embody the
    "external action" being performed), and an optional message to
    print.
    """
    if msg is None:
        msg = "%s%r" % (func.__name__, args)
        if msg[-2:] == ',)':        # correct for singleton tuple
            msg = msg[0:-2] + ')'

    log.info(msg)
    if not dry_run:
        func(*args)

def strtobool (val):
    """Convert a string representation of truth to true (1) or false (0).

    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
    'val' is anything else.
    """
    val = val.lower()
    if val in ('y', 'yes', 't', 'true', 'on', '1'):
        return 1
    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
        return 0
    else:
        raise ValueError("invalid truth value %r" % (val,))

def byte_compile (py_files,
                  optimize=0, force=0,
                  prefix=None, base_dir=None,
                  verbose=1, dry_run=0,
                  direct=None):
    """Byte-compile a collection of Python source files to .pyc
    files in a __pycache__ subdirectory.  'py_files' is a list
    of files to compile; any files that don't end in ".py" are silently
    skipped.  'optimize' must be one of the following:
      0 - don't optimize
      1 - normal optimization (like "python -O")
      2 - extra optimization (like "python -OO")
    If 'force' is true, all files are recompiled regardless of
    timestamps.

    The source filename encoded in each bytecode file defaults to the
    filenames listed in 'py_files'; you can modify these with 'prefix' and
    'basedir'.  'prefix' is a string that will be stripped off of each
    source filename, and 'base_dir' is a directory name that will be
    prepended (after 'prefix' is stripped).  You can supply either or both
    (or neither) of 'prefix' and 'base_dir', as you wish.

    If 'dry_run' is true, doesn't actually do anything that would
    affect the filesystem.

    Byte-compilation is either done directly in this interpreter process
    with the standard py_compile module, or indirectly by writing a
    temporary script and executing it.  Normally, you should let
    'byte_compile()' figure out to use direct compilation or not (see
    the source for details).  The 'direct' flag is used by the script
    generated in indirect mode; unless you know what you're doing, leave
    it set to None.
    """

    # Late import to fix a bootstrap issue: _posixsubprocess is built by
    # setup.py, but setup.py uses distutils.
    import subprocess

    if sys.dont_write_bytecode:
        raise DistutilsByteCompileError('byte-compiling is disabled.')

    # First, if the caller didn't force us into direct or indirect mode,
    # figure out which mode we should be in.  We take a conservative
    # approach: choose direct mode *only* if the current interpreter is
    # or -OO), we don't know which level of optimization this
    # interpreter is running with, so we can't do direct
    # byte-compilation and be certain that it's the right thing.  Thus,
    # always compile indirectly if the current interpreter is in either
    # optimize mode, or if either optimization level was requested by
    # the caller.
    if direct is None:
        direct = (__debug__ and optimize == 0)

    # "Indirect" byte-compilation: write a temporary script and then
    # run it with the appropriate flags.
    if not direct:
        try:
            from tempfile import mkstemp
            (script_fd, script_name) = mkstemp(".py")
        except ImportError:
            from tempfile import mktemp
            (script_fd, script_name) = None, mktemp(".py")
        log.info("writing byte-compilation script '%s'", script_name)
        if not dry_run:
            if script_fd is not None:
                script = os.fdopen(script_fd, "w")
            else:
                script = open(script_name, "w")

            script.write("""\
from distutils.util import byte_compile
files = [
""")

            # XXX would be nice to write absolute filenames, just for
            # safety's sake (script should be more robust in the face of
            # chdir'ing before running it).  But this requires abspath'ing
            # 'prefix' as well, and that breaks the hack in build_lib's
            # 'byte_compile()' method that carefully tacks on a trailing
            # slash (os.sep really) to make sure the prefix here is "just
            # right".  This whole prefix business is rather delicate -- the
            # problem is that it's really a directory, but I'm treating it
            # as a dumb string, so trailing slashes and so forth matter.

            script.write(",\n".join(map(repr, py_files)) + "]\n")
            script.write("""
byte_compile(files, optimize=%r, force=%r,
             prefix=%r, base_dir=%r,
             verbose=%r, dry_run=0,
             direct=1)
""" % (optimize, force, prefix, base_dir, verbose))

            script.close()

        cmd = [sys.executable]
        cmd.extend(subprocess._optim_args_from_interpreter_flags())
        cmd.append(script_name)
        spawn(cmd, dry_run=dry_run)
        execute(os.remove, (script_name,), "removing %s" % script_name,
                dry_run=dry_run)

    # "Direct" byte-compilation: use the py_compile module to compile
    # right here, right now.  Note that the script generated in indirect
    # mode simply calls 'byte_compile()' in direct mode, a weird sort of
    # cross-process recursion.  Hey, it works!
    else:
        from py_compile import compile

        for file in py_files:
            if file[-3:] != ".py":
                # This lets us be lazy and not filter filenames in
                continue

            # Terminology from the py_compile module:
            #   cfile - byte-compiled file
            #   dfile - purported source filename (same as 'file' by default)
            if optimize >= 0:
                opt = '' if optimize == 0 else optimize
                cfile = importlib.util.cache_from_source(
                    file, optimization=opt)
            else:
                cfile = importlib.util.cache_from_source(file)
            dfile = file
            if prefix:
                if file[:len(prefix)] != prefix:
                    raise ValueError("invalid prefix: filename %r doesn't start with %r"
                           % (file, prefix))
                dfile = dfile[len(prefix):]
            if base_dir:
                dfile = os.path.join(base_dir, dfile)

            cfile_base = os.path.basename(cfile)
            if direct:
                if force or newer(file, cfile):
                    log.info("byte-compiling %s to %s", file, cfile_base)
                    if not dry_run:
                        compile(file, cfile, dfile)
                else:
                    log.debug("skipping byte-compilation of %s to %s",
                              file, cfile_base)

def rfc822_escape (header):
    """Return a version of the string escaped for inclusion in an
    RFC-822 header, by ensuring there are 8 spaces space after each newline.
    """
    lines = header.split('\n')
    sep = '\n' + 8 * ' '
    return sep.join(lines)

# 2to3 support

def run_2to3(files, fixer_names=None, options=None, explicit=None):
    """Invoke 2to3 on a list of Python files.
    The files should all come from the build area, as the
    modification is done in-place. To reduce the build time,
    only files modified since the last invocation of this
    function should be passed in the files argument."""

    if not files:
        return

    # Make this class local, to delay import of 2to3
    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
    class DistutilsRefactoringTool(RefactoringTool):
        def log_error(self, msg, *args, **kw):
            log.error(msg, *args)

        def log_message(self, msg, *args):
            log.info(msg, *args)

        def log_debug(self, msg, *args):
            log.debug(msg, *args)

    if fixer_names is None:
        fixer_names = get_fixers_from_package('lib2to3.fixes')
    r = DistutilsRefactoringTool(fixer_names, options=options)
    r.refactor(files, write=True)

def copydir_run_2to3(src, dest, template=None, fixer_names=None,
                     options=None, explicit=None):
    """Recursively copy a directory, only copying new and changed files,
    running run_2to3 over all newly copied Python modules afterward.

    If you give a template string, it's parsed like a MANIFEST.in.
    """
    from distutils.dir_util import mkpath
    from distutils.file_util import copy_file
    from distutils.filelist import FileList
    filelist = FileList()
    curdir = os.getcwd()
    os.chdir(src)
    try:
        filelist.findall()
    finally:
        os.chdir(curdir)
    filelist.files[:] = filelist.allfiles
    if template:
        for line in template.splitlines():
            line = line.strip()
            if not line: continue
            filelist.process_template_line(line)
    copied = []
    for filename in filelist.files:
        outname = os.path.join(dest, filename)
        mkpath(os.path.dirname(outname))
        res = copy_file(os.path.join(src, filename), outname, update=1)
        if res[1]: copied.append(outname)
    run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
             fixer_names=fixer_names, options=options, explicit=explicit)
    return copied

class Mixin2to3:
    '''Mixin class for commands that run 2to3.
    To configure 2to3, setup scripts may either change
    the class variables, or inherit from individual commands
    to override how 2to3 is invoked.'''

    # provide list of fixers to run;
    # defaults to all from lib2to3.fixers
    fixer_names = None

    # options dictionary
    options = None

    # list of fixers to invoke even though they are marked as explicit
    explicit = None

    def run_2to3(self, files):
        return run_2to3(files, self.fixer_names, self.options, self.explicit)


    h, w = arr.shape
    out_h = math.floor(h * scale[0])
    out_w = math.floor(w * scale[1])
    out = np.empty((out_h, out_w))
    for y in range(out_h):
        for x in range(out_w):
            in_y = math.floor(y / scale[0])
            in_x = math.floor(x / scale[1])
            out[y, x] = arr[in_y, in_x]
    return out

"""Upsampling in python"""
import math
import numpy as np

def upsample_nearest(arr, scale):
    """ Populate the array by scale factor"""

def upsampling_python(data, scale, layout='NCHW'):
    """ Python version of scaling using nearest neighbour """

    ishape = data.shape
    if layout == 'NCHW':
        oshape = (ishape[0], ishape[1], math.floor(ishape[2]*scale[0]), math.floor(ishape[3]*scale[1]))
        output_np = np.zeros(oshape, dtype=data.dtype)
        for b in range(oshape[0]):
            for c in range(oshape[1]):
                output_np[b, c, :, :] = upsample_nearest(data[b, c, :, :], scale)
        return output_np
    if layout == 'NHWC':
        oshape = (ishape[0], math.floor(ishape[1]*scale[0]), math.floor(ishape[1]*scale[1]), ishape[3])
        output_np = np.zeros(oshape, dtype=data.dtype)
        for b in range(oshape[0]):
            for c in range(oshape[3]):
                output_np[b, :, :, c] = upsample_nearest(data[b, :, :, c], scale)
        return output_np
    raise ValueError("not support this layout {} yet".format(layout))


    modelNew = VGG16Model()
    if x is not None:
        cnnfeaturesX = modelNew.predict(x)
        return cnnfeaturesX

"""
Created on Sun Apr 21 12:13:10 2018

@author: fubao
"""
#load the pretrained model weights ; transferred learning features

import numpy as np

import feature_utility as fu
import cv2
from matplotlib import pyplot as plt

from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D

from keras.utils import plot_model
from keras import backend as K

import modifiedVGG

from keras.applications.vgg16 import VGG16

from keras.applications.vgg16 import preprocess_input
from keras.models import Model
from keras.preprocessing import image
from keras.layers.normalization import BatchNormalization
from CNNActivation import get_activations, display_activations

from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw

def preprocessing(img, size=(256, 256)):
    #print ("preprocessing img file", img)

    img = cv2.imread(img, 0)
    img = cv2.resize(img, size).astype(np.float32)
    img = np.expand_dims(img, axis=0)
    return img

def modifiedVGG16Model(weights_path=None, shape=(256, 256)):
    '''
    no output softmax
    '''
    model = Sequential()
    model.add(ZeroPadding2D((1,1), input_shape=shape))
    model.add(Convolution2D(32, 3, 3, activation='relu'))           # 1
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(32, 3, 3, activation='relu'))           # 3
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(64, 3, 3, activation='relu'))           # 6
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(64, 3, 3, activation='relu'))          # 8
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3, activation='relu'))         # 11
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3, activation='relu'))         # 13
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3, activation='relu'))         # 15
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    '''
    model.add(Flatten())
    model.add(Dense(1024, activation='relu'))                      # 18
    model.add(Dropout(0.5))
    model.add(Dense(512, activation='relu'))                       # 20
    model.add(Dropout(0.5))
    '''
    print ("modifiedVGG16Model Create model successfully")

    return model

def VGG16Model():
    '''
    use pretrained VGG16 model
    '''

    #load vgg16 without dense layer and with theano dim ordering
    base_model = VGG16(weights = 'imagenet', include_top = False, input_shape = (3, 256, 256))

    #number of classes in your dataset e.g. 20
    x = Flatten()(base_model.output)
    x = Dense(4096, activation='relu')(x)
    x = Dropout(0.5)(x)
    x = BatchNormalization()(x)

    #Add a layer where input is the output of the  second last layer

    #Then create the corresponding model
    newModel = Model(input=base_model.input, output=x)
    newModel.summary()

    return newModel

def extractModifiedVGG16toArray(x, shape):

    '''
    get features from an  numpy array sets from the cnn pretrained model
    '''

def extractModifiedVGGSingleImages(inputImage, shape):

    '''
    get loaded weight from previous model;
    not use the last layer (softmax)
    apply on the single image to get features
    '''
    weights_path = "my_model_weights_83.h5"

    model = modifiedVGG.VGG_16(weights_path=weights_path, shape=shape)

    #print ("first layer weight: ",len(weights), np.asarray(weights).shape)
    modelNew = modifiedVGG16Model(weights_path=None, shape=shape)
    modelNew.set_weights(model.get_weights()[:-1])
    print ("modelNew summary: ", modelNew.summary())

    weights = modelNew.layers[-2].get_weights()
    print ("firsttt layer weight: ",len(weights), np.asarray(weights).shape)

    if inputImage is not None:
        img = preprocessing(inputImage)
        x = np.expand_dims(img, axis=0)
        print ("XX: ", x.shape)

        x = np.asarray(x)
        # do mean and standardlization
        x -= np.mean(x, axis=0)
        x /= np.std(x, axis=0)
        print ("x: ", x.shape)

def extractModifiedVGGArray(x, shape):

    '''
    get features from an  numpy array sets from the cnn pretrained model vGG16
    '''

    print ("extractModifiedVGGArray x  shape", x.shape, shape)

    weights_path = "my_model_weights_83.h5"

    model = modifiedVGG.VGG_16(weights_path=weights_path, shape=shape)

    #print ("first layer weight: ",len(weights), np.asarray(weights).shape)
    modelNew = modifiedVGG16Model(weights_path = None, shape=shape)

    modelNew.set_weights(model.get_weights()[:-5])
    print ("modelNew summary: ", modelNew.summary())

    if x is not None:
        cnnfeaturesX = modelNew.predict(x)
        return cnnfeaturesX
    return None

def layer_to_visualize(img_to_visualize, model, layer):
    inputs = [K.learning_phase()] + model.inputs

    _convout1_f = K.function(inputs, [layer.output])
    def convout1_f(X):
        # The [0] is to disable the training phase flag
        return _convout1_f([0] + [X])

    convolutions = convout1_f(img_to_visualize)
    convolutions = np.squeeze(convolutions)

    print ('Shape of conv:', convolutions.shape)

    n = convolutions.shape[0]
    n = int(np.ceil(np.sqrt(n)))

    # Visualization of each filter of the layer
    fig = plt.figure(figsize=(12,8))
    for i in range(len(convolutions)):
        ax = fig.add_subplot(n,n,i+1)
        ax.imshow(convolutions[i], cmap='gray')

def visualLayersOutput(testImage):

    img = preprocessing(testImage, size=(256, 256))
    x = np.expand_dims(img, axis=0)
    x = np.asarray(x)

    print (" visualLayersOutput, x shape: ", x.shape)

    weights_path = "my_model_weights_83.h5"
    model = modifiedVGG.VGG_16(weights_path=weights_path, shape=(1,256,256))

    #print ("first layer weight: ",len(weights), np.asarray(weights).shape)
    modelNew = modifiedVGG16Model(weights_path=None, shape=(1, 256,256))
    modelNew.set_weights(model.get_weights()[:-1])

    print (" visualLayersOutput, modelNew: ")

    a = get_activations(modelNew, x, print_shape_only=True)  # with just one sample.
    display_activations(a)

    '''
    weights_path = "my_model_weights_83.h5"
    model = modifiedVGG.VGG_16(weights_path=weights_path, shape=(1, 256,256))

    #print ("first layer weight: ",len(weights), np.asarray(weights).shape)
    modelNew = modifiedVGG16Model(weights_path=None, shape=(1, 256,256))
    modelNew.set_weights(model.get_weights()[:-1])

    img = preprocessing(testImage, size=(256, 256))
    x = np.expand_dims(img, axis=0)
    x = np.asarray(x)

    print (" visualLayersOutput, x shape: ", x.shape)

    desiredLayers = [1,5]
    desiredOutputs = [modelNew.layers[i].output for i in desiredLayers]

    #alternatively, you can use cnnModel.get_layer('layername').output for that

    newModel = Model(modelNew.inputs, desiredOutputs)
    output = newModel.predict(x)
    print("newModel.predict(x): ", output.shape)

    plt.imshow(output)
    '''

def textOnImage(image):
    font = ImageFont.truetype(image,25)
    img=Image.new("RGBA", (200,200),(120,20,20))
    draw = ImageDraw.Draw(img)
    draw.text((0, 0),"This is a test",(255,255,0),font=font)
    draw = ImageDraw.Draw(img)
    draw = ImageDraw.Draw(img)
    img.save("a_test.png")

if __name__ == "__main__":
    '''
    testImage = "../dataSet/jaffe/Angry/KA.AN3.41.tiff"

    model = VGG16Model()
    img = image.load_img(testImage, target_size=(256, 256))
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)
    cnnfeaturesX = model.predict(x)
    print ("shape: ",cnnfeaturesX.shape, cnnfeaturesX)
    '''
    testImage = "../dataSet/jaffe/Angry/KA.AN3.41.tiff"
    textOnImage(testImage)


self.assertMarkdownRenders(
            self.dedent(
                """
                I would like to tell you about the [code of

                conduct][] we are using in this project.

                [code of conduct]: https://github.com/Python-Markdown/markdown/blob/master/CODE_OF_CONDUCT.md
                """
            ),
            '
I would like to tell you about the [code of
\n'
            '
conduct][] we are using in this project.
'
        )

"""
Python Markdown

A Python implementation of John Gruber's Markdown.

Documentation: http://example.com
GitHub: http://example.com
PyPI: http://example.com

Started by Manfred Stienstra (http://example.com).
Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org).
Currently maintained by Waylan Limberg (http://example.com),
Dmitry Shachnev (http://example.com) and Isaac Muse (http://example.com).

"""

from markdown.test_tools import TestCase

class TestInlineLinks(TestCase):

    def test_nested_square_brackets(self):
        self.assertMarkdownRenders(
            """[Text[[[[[[[]]]]]]][]](http://link.com) more text""",
            """
Text[[[[[[[]]]]]]][] more text
"""
        )

    def test_nested_round_brackets(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/(((((((()))))))())) more text""",
            """
Text more text
"""
        )

    def test_uneven_brackets_with_titles1(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/("title") more text""",
            """
Text more text
"""
        )

    def test_uneven_brackets_with_titles2(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/('"title") more text""",
            """
Text more text
"""
        )

    def test_uneven_brackets_with_titles3(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/("title)") more text""",
            """
Text more text
"""
        )

    def test_uneven_brackets_with_titles4(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/( "title") more text""",
            """
Text more text
"""
        )

    def test_uneven_brackets_with_titles5(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/( "title)") more text""",
            """
Text more text
"""
        )

    def test_mixed_title_quotes1(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/'"title") more text""",
            """
Text more text
"""
        )

    def test_mixed_title_quotes2(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/"'title') more text""",
            """
Text more text
"""
        )

    def test_mixed_title_quotes3(self):
        self.assertMarkdownRenders(
            """[Text](http://example.com spaces'"and quotes" 'and title') more text""",
            """
"""
            """Text more text
"""
        )

    def test_mixed_title_quotes4(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/with spaces'"and quotes" 'and title") more text""",
            """
Text more text
"""
        )

    def test_mixed_title_quotes5(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/with spaces '"and quotes" 'and title') more text""",
            """
"""
            """Text more text
"""
        )

    def test_mixed_title_quotes6(self):
        self.assertMarkdownRenders(
            """[Text](http://link.com/with spaces "and quotes" 'and title') more text""",
            """
"""
            """Text more text
"""
        )

    def test_single_quote(self):
        self.assertMarkdownRenders(
            """[test](link"notitle)""",
            """
test
"""
        )

    def test_angle_with_mixed_title_quotes(self):
        self.assertMarkdownRenders(
            """[Text](<http://link.com/with spaces '"and quotes"> 'and title') more text""",
            """
"""
            """Text more text
"""
        )

    def test_amp_in_url(self):
        """Test amp in URLs."""

        self.assertMarkdownRenders(
            '[link](http://example.com)',
            '
link
'
        )
        self.assertMarkdownRenders(
            '[title](http://example.com/?a=1&b=2)',
            '
title
'
        )
        self.assertMarkdownRenders(
            '[title](http://example.com/?a=1&b=2)',
            '
title
'
        )

class TestReferenceLinks(TestCase):

    def test_ref_link(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com
                """
            ),
            """
Text
"""
        )

    def test_ref_link_angle_brackets(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: <http://example.com>
                """
            ),
            """
Text
"""
        )

    def test_ref_link_no_space(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]:http://example.com
                """
            ),
            """
Text
"""
        )

    def test_ref_link_angle_brackets_no_space(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]:<http://example.com>
                """
            ),
            """
Text
"""
        )

    def test_ref_link_angle_brackets_title(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: <http://example.com> "title"
                """
            ),
            """
Text
"""
        )

    def test_ref_link_title(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com "title"
                """
            ),
            """
Text
"""
        )

    def test_ref_link_angle_brackets_title_no_space(self):
        # TODO: Maybe reevaluate this?
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: <http://example.com>"title"
                """
            ),
            """
Text
"""
        )

    def test_ref_link_title_no_space(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com"title"
                """
            ),
            """
Text
"""
        )

    def test_ref_link_single_quoted_title(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com 'title'
                """
            ),
            """
Text
"""
        )

    def test_ref_link_title_nested_quote(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com "title'"
                """
            ),
            """
Text
"""
        )

    def test_ref_link_single_quoted_title_nested_quote(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com 'title"'
                """
            ),
            """
Text
"""
        )

    def test_ref_link_override(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]: http://example.com 'ignore'
                [Text]: https://example.com 'override'
                """
            ),
            """
Text
"""
        )

    def test_ref_link_title_no_blank_lines(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]
                [Text]: http://example.com "title"
                [Text]
                """
            ),
            self.dedent(
                """

Text

Text

                """
            )
        )

    def test_ref_link_multi_line(self):
        self.assertMarkdownRenders(
            self.dedent(
                """
                [Text]

                [Text]:
                    http://example.com
                    "title"
                """
            ),
            """
Text
"""
        )

    def test_reference_newlines(self):
        """Test reference id whitespace cleanup."""

        self.assertMarkdownRenders(
            self.dedent(
                """
                Two things:

                 - I would like to tell you about the [code of
                   conduct][] we are using in this project.
                 - Only one in fact.

                [code of conduct]: http://example.com
                """
            ),
            '
Two things:
\n
\n
I would like to tell you about the '
            'code of\n'
            '   conduct we are using in this project.
\n
Only one in fact.
\n
'
        )

    def test_reference_across_blocks(self):
        """Test references across blocks."""


        # We import our signals here to ensure they're attached to the correct
        from open_connect.accounts.signals import *

"""Application Configuration for the Accounts application"""
from django.apps import AppConfig

class AccountsConfig(AppConfig):
    """App configuration for the Accounts application"""
    name = 'open_connect.accounts'
    verbose_name = 'Accounts'

    # pylint: disable=no-self-use
    def ready(self):
        """Functionality to execute with the app is ready"""


        yDistance = (lat2 - lat1) * nauticalMilePerLat
        xDistance = ((math.cos(lat1 * rad) + math.cos(lat2 * rad)) *
                     (lon2 - lon1) * (nauticalMilePerLongitude / 2))

        distance = math.sqrt(yDistance**2 + xDistance**2)

        return distance * metersPerNauticalMile

import os
import urllib
import re
import time
import math
import sqlite3
import sys
import gzip
import logging
import csv

from datetime import datetime, timedelta
from pkg_resources import resource_filename

try:
    from urllib.request import urlopen
    from urllib.error import HTTPError
except ImportError:
    from urllib2 import urlopen, HTTPError

"""Constants
"""
nauticalMilePerLat = 60.00721
nauticalMilePerLongitude = 60.10793
rad = math.pi / 180.0
metersPerNauticalMile = 1852

STATION_INFO_COLS = ["sid", "type", "name", "lat", "lon", "nth", "distance"]

USAF_WBAN_DATA = [('TEMP', 24, 30), ('DEWP', 35, 41), ('SLP', 46, 52),
                  ('STP', 57, 63), ('VISIB', 68, 73), ('AWND', 78, 83),
                  ('MXSPD', 88, 93), ('GUST', 95, 100), ('TMAX', 102, 108),
                  ('MAXF', 108, 109), ('TMIN', 110, 116), ('MINF', 116, 117),
                  ('PRCP', 118, 123), ('PRCPF', 123, 124), ('SNWD', 125, 130),
                  ('FRSHTT', 132, 138)]

def download(url, local):
    print("Downloading '{:s}'...".format(url))
    start = time.time()
    response = urlopen(url)
    content = response.read()
    with open(local, 'wb') as f:
        f.write(content)
    elapse = time.time() - start
    print("Elapse time: {:f} seconds".format(elapse))

def build_ghcn_database(args, year, dbname):
    conn = None
    try:
        local = os.path.join(args.dbpath, "{:s}.csv.gz".format(year))
        if not os.path.exists(local):
            if not os.path.exists(args.dbpath):
                os.makedirs(args.dbpath)
            url = "ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/by_year/{:s}.csv.gz".format(year)
            download(url, local)
        conn = sqlite3.connect(dbname)
        c = conn.cursor()
        c.execute("""CREATE TABLE IF NOT EXISTS `ghcn_{:s}` (
`id` VARCHAR(12) NOT NULL,
`date` VARCHAR(8) NOT NULL,
`element` VARCHAR(4) NULL,
`value` VARCHAR(6) NULL,
`m_flag` VARCHAR(1) NULL,
`q_flag` VARCHAR(1) NULL,
`s_flag` VARCHAR(1) NULL,
`obs_time` VARCHAR(4) NULL)""".format(year))

        c.execute("""create index if not exists idx_id_time
on ghcn_{:s} (id, date)""".format(year))

        c.execute("PRAGMA journal_mode = OFF")
        c.execute("PRAGMA synchronous = OFF")
        # Fastest way if no memory limit
        c.execute("PRAGMA cache_size = 1000000")
        print("Importing...")
        start = time.time()
        with gzip.open(local, 'rb') as f:
            reader = csv.reader(f)
            c.executemany("""INSERT OR IGNORE INTO ghcn_{:s}
(id, date, element, value, m_flag, q_flag, s_flag, obs_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""".format(year), reader)
        elapse = time.time() - start
        print("Import time: {:f} seconds".format(elapse))
    except Exception as e:
        logging.error(str(e))
        raise
    finally:
        if conn:
            print("Commiting...")
            start = time.time()
            conn.commit()
            conn.close()
            elapse = time.time() - start
            print("Commit time: {:f} seconds".format(elapse))

class WeatherByZip(object):

    not_found_list = []

    def get_stations(self):
        """Returns all stations from database
        """
        conn = sqlite3.connect(self.args.zip2ws_db)
        c = conn.cursor()
        c.execute("select rowid, id, name, lat, lon, type from stations")
        stations = []
        for r in c:
            stations.append(r)
        conn.close()
        return stations

    def __init__(self, args):
        self.args = args
        try:
            with open(args.columns, 'rb') as f:
                self.output_columns = [r.strip() for r in f.readlines()
                                       if r[0] != '#']
        except:
            args.columns = resource_filename(__name__, args.columns)
            with open(args.columns, 'rb') as f:
                self.output_columns = [r.strip() for r in f.readlines()
                                       if r[0] != '#']

        self.stations = self.get_stations()
        logging.info('Number of Stations = {:d}'.format(len(self.stations)))

    def search_weather_data(self, dist, data):
        year = "{:04d}" .format(data['year'])
        month = "{:02d}".format(data['month'])
        day = "{:02d}".format(data['day'])
        logging.debug("Date: {:s}/{:s}/{:s}".format(year, month, day))
        found = 0
        nth = 0
        raw = []
        values = {}
        current_year = ""
        orders = self.output_columns
        for s in dist:
            if self.args.nth > 0 and nth >= self.args.nth:
                logging.info("Reach maximum n-th stations: {:d}".format(nth))
                return values
            if self.args.distance > 0 and s[0] >= self.args.distance * 1000:
                logging.info("Reach maximum distance: {:d}"
                             .format(self.args.distance))
                return values
            nth += 1
            values['nth'] = nth
            sid = s[1]
            stype = s[2]
            values['sid'] = sid
            values['type'] = stype
            values['name'] = s[3]
            values['lat'] = s[4]
            values['lon'] = s[5]
            values['distance'] = s[0]
            if stype == 'GHCND':
                if not self.args.uses_sqlite:
                    datadir = './data/ghcn-daily/all/'
                    if not os.path.exists(datadir):
                        os.makedirs(datadir)
                    datafile = datadir + '{:s}.dly'.format(sid)
                    urlfile = ('http://example.com/'
                               'all/{:s}.dly'.format(sid))
                    if not self.download_data_file(urlfile, datafile):
                        continue
                    search = sid + year + month
                    match = False
                    with open(datafile, 'rb') as df:
                        for l in df:
                            if (l[:17] == search):
                                element = l[17:21]
                                offset = 21 + int(day) * 8 - 8
                                value = l[offset:offset + 5]
                                mflag = l[offset + 5:offset + 6]
                                qflag = l[offset + 6:offset + 7]
                                sflag = l[offset + 7:offset + 8]
                                logging.debug("<{:s}> <{:s}> <{:s}> <{:s}>"
                                              " <{:s}>".format(element, value,
                                                               mflag, qflag,
                                                               sflag))
                                match = True
                                if value != '-9999' and (element in orders):
                                    if not values.get(element):
                                        values[element] = value
                                        found += 1
                                        raw.append(l)
                            elif match:
                                break
                else:
                    if current_year != year:
                        current_year = year
                        dbname = os.path.join(self.args.dbpath,
                                              "ghcn_{:s}.sqlite3".format(year))
                        if not os.path.exists(dbname):
                            build_ghcn_database(self.args, year, dbname)
                        conn2 = sqlite3.connect(dbname)
                        c2 = conn2.cursor()
                        # FIXME: try to optimize with these options
                        c2.execute("PRAGMA journal_mode = MEMORY")
                        c2.execute("PRAGMA synchronous = OFF")
                        c2.execute("PRAGMA temp_store = MEMORY")
                        c2.execute("PRAGMA cache_size = 500000")
                    ghcn_values = self.get_ghcn_data(c2, sid, year, month, day)
                    for element in ghcn_values:
                        value = ghcn_values[element]
                        if value != '-9999' and (element in orders):
                            if not values.get(element):
                                found += 1
                                values[element] = value
            elif stype == 'USAF-WBAN':
                datadir = './data/gsod/{:s}/'.format(year)
                if not os.path.exists(datadir):
                    os.makedirs(datadir)
                datafile = datadir + '{:s}-{:s}.op.gz'.format(sid, year)
                urlfile = ('http://example.com/'
                           '{:s}/{:s}-{:s}.op.gz'.format(year, sid, year))
                if not self.download_data_file(urlfile, datafile):
                    continue
                search = year + month + day
                with gzip.open(datafile, 'rb') as df:
                    for l in df:
                        if l[14:22] == search:
                            match = False
                            for d in USAF_WBAN_DATA:
                                element = d[0]
                                value = l[d[1]:d[2]]
                                if element in ['TMAX', 'TMIN']:
                                    if value != '9999.9':
                                        value = self.f2c(float(value)) * 10
                                elif element in ['AWND']:
                                    if value != '999.9':
                                        value = self.kn2ms(float(value)) * 10
                                if (not values.get(element) and
                                   (element in orders)):
                                    found += 1
                                    values[element] = value
                                    match = True
                            if match:
                                raw.append(l)
                            break
            if found >= len(orders):
                break
        logging.debug(values)
        # For debug only
        return values

    def search(self, search):
        zipcode = '0' * (5 - len(search['zip'])) + search['zip']

        logging.info("Search for: '{:s}'".format(zipcode))

        from_date = datetime(year=search['from.year'],
                             month=search['from.month'],
                             day=search['from.day'])
        to_date = datetime(year=search['to.year'],
                           month=search['to.month'],
                           day=search['to.day'])
        columns = (['uniqid', 'zip', 'year', 'month', 'day'] +
                   STATION_INFO_COLS + self.output_columns)
        results = []
        conn = sqlite3.connect(self.args.zip2ws_db)
        c = conn.cursor()
        c.execute('select zipcode, lat, lon, gm_lat, gm_lon, rowid from zip'
                  ' where zipcode = ?', (zipcode,))
        r = c.fetchone()
        dist = self.sort_stations(zipcode, r, self.stations)
        while from_date <= to_date:
            data = {}
            data['uniqid'] = search['uniqid']
            data['zip'] = zipcode
            data['year'] = from_date.year
            data['month'] = from_date.month
            data['day'] = from_date.day
            if dist and len(dist) > 0:
                wdata = self.search_weather_data(dist, data)
                data.update(wdata)
            results.append(data)
            from_date += timedelta(days=1)
        return results

    def download_data_file(self, url, file):
        """Download weather data file from server if it not exist in local directory
        """
        if url in self.not_found_list:
            logging.warn("This URL no data on server {:s}".format(url))
            return False
        if not os.path.exists(file):
            retry = 0
            while True:
                try:
                    logging.info("Downloading '{:s}'".format(url))
                    urllib.urlretrieve(url, file)
                    return True
                except Exception as e:
                    m = re.match(".*\s(\d\d\d)\s.*", str(e))
                    if m:
                        logging.warn("FTP server error code = {:s}"
                                     .format(m.group(1)))
                        if m.group(1) == '550':
                            self.not_found_list.append(url)
                            return False
                    logging.warn("Unknown FTP server error = {:s}"
                                 .format(str(e)))
                    if retry < 5:
                        retry += 1
                        logging.info("Retry #{:d}: waiting...({:d}s)"
                                     .format(retry, retry * 10))
                        time.sleep(retry * 10)
                    else:
                        logging.warn("Cannot download data from URL = {:s}"
                                     .format(url))
                        return False
        return True

    def metersGeoDistance(self, lat1, lon1, lat2, lon2):
        """Returns calculate distance between two lat lons in meters
        """

    def f2c(self, f):
        """Convert Fahrenheit to Celsius
        """
        return (f - 32) * 5.0/9.0

    def kn2ms(self, kn):
        """Convert Knots to m/s
        """
        return 0.51444 * kn

    def sort_stations(self, zipcode, r, stations):
        """Returns sorted stations list by distance
        """
        if r is None:
            logging.warn("zipcode = {:s} not found".format(zipcode))
            return None
        lat1 = r[3]
        if lat1 is None:
            lat1 = r[1]
        lon1 = r[4]
        if lon1 is None:
            lon1 = r[2]
        if lat1 == '' or lon1 == '':
            logging.warn("not lat/lon for zipcode = {:s}".format(zipcode))
            return None
        lat1 = float(lat1)
        lon1 = float(lon1)
        dist = []
        for s in stations:
            id = s[1]
            name = s[2]
            stype = s[5]
            if s[3] is None or s[4] is None:
                continue
            lat2 = float(s[3])
            lon2 = float(s[4])
            try:
                distance = self.metersGeoDistance(lat1, lon1, lat2, lon2)
            except:
                distance = sys.maxint
            dist.append((int(distance), id, stype, name, lat2, lon2))
        return sorted(dist)

    def get_ghcn_data(self, cursor, sid, year, month, day):
        cursor.execute("select * from ghcn_{:s} where id = ? and date = ?"
                       .format(year), (sid, '{:02d}{:02d}{:02d}'
                                       .format(int(year), int(month),
                                               int(day))))
        values = {}
        for r in cursor:
            values[r[2]] = r[3]
        return values