    with _replace_file(path) as tmp_io:
        tmp_io.truncate(size)
        LOGGER.info('Created sparse file: %r', tmp_io.name)

# The Qubes OS Project, http://example.com

''' Driver for handling VM images as files, without any device-mapper
    involvement. A reflink-capable filesystem is strongly recommended,
    but not required.
'''

import asyncio
import collections
import errno
import fcntl
import functools
import glob
import logging
import os
import platform
import subprocess
import tempfile
from contextlib import suppress

import qubes.storage
import qubes.utils

LOGGER = logging.getLogger('qubes.storage.reflink')

LOOP_SET_CAPACITY = 0x4C07

FICLONE = {
    'x86_64':  0x40049409,
    'ppc64le': 0x80049409,
}[platform.machine()]

def _coroutinized(function):
    ''' Wrap a synchronous function in a coroutine that runs the
        function via the event loop's ThreadPool-based default
        executor.
    '''
    @functools.wraps(function)
    async def wrapper(*args, **kwargs):
        return await asyncio.get_event_loop().run_in_executor(
            None, functools.partial(function, *args, **kwargs))
    return wrapper

class ReflinkPool(qubes.storage.Pool):
    driver = 'file-reflink'
    _known_dir_path_prefixes = ['appvms', 'vm-templates']

    def __init__(self, *, name, revisions_to_keep=1,
                 dir_path, setup_check=True, ephemeral_volatile=False):
        super().__init__(name=name, revisions_to_keep=revisions_to_keep,
                         ephemeral_volatile=ephemeral_volatile)
        self._setup_check = qubes.property.bool(None, None, setup_check)
        self._volumes = {}
        self.dir_path = os.path.abspath(dir_path)

    @_coroutinized
    def setup(self):  # pylint: disable=invalid-overridden-method
        created = _make_dir(self.dir_path)
        if self._setup_check and not is_supported(self.dir_path):
            if created:
                _remove_empty_dir(self.dir_path)
            raise qubes.storage.StoragePoolException(
                'The filesystem for {!r} does not support reflinks. If you'
                ' can live with VM startup delays and wasted disk space, pass'
                ' the "setup_check=False" option.'.format(self.dir_path))
        for dir_path_prefix in self._known_dir_path_prefixes:
            _make_dir(os.path.join(self.dir_path, dir_path_prefix))
        return self

    def init_volume(self, vm, volume_config):
        # Fail closed on any strange VM dir_path_prefix, just in case
        # /etc/udev/rules.d/00-qubes-ignore-devices.rules needs update
        assert vm.dir_path_prefix in self._known_dir_path_prefixes, \
               'Unknown dir_path_prefix {!r}'.format(vm.dir_path_prefix)

        volume_config['pool'] = self
        if 'revisions_to_keep' not in volume_config:
            volume_config['revisions_to_keep'] = self.revisions_to_keep
        if 'vid' not in volume_config:
            volume_config['vid'] = os.path.join(
                vm.dir_path_prefix, vm.name, volume_config['name'])
        volume = ReflinkVolume(**volume_config)
        self._volumes[volume_config['vid']] = volume
        return volume

    def list_volumes(self):
        return list(self._volumes.values())

    def get_volume(self, vid):
        return self._volumes[vid]

    async def destroy(self):
        pass

    @property
    def config(self):
        return {
            'name': self.name,
            'dir_path': self.dir_path,
            'driver': ReflinkPool.driver,
            'revisions_to_keep': self.revisions_to_keep,
            'ephemeral_volatile': self.ephemeral_volatile,
        }

    @property
    def size(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * statvfs.f_blocks

    @property
    def usage(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)

    def included_in(self, app):
        ''' Check if there is pool containing this one - either as a
        filesystem or its LVM volume'''
        return qubes.storage.search_pool_containing_dir(
            [pool for pool in app.pools.values() if pool is not self],
            self.dir_path)

class ReflinkVolume(qubes.storage.Volume):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
        self._path_clean = self._path_vid + '.img'
        self._path_dirty = self._path_vid + '-dirty.img'
        self._path_import = self._path_vid + '-import.img'
        self.path = self._path_dirty

    @qubes.storage.Volume.locked
    @_coroutinized
    def create(self):  # pylint: disable=invalid-overridden-method
        self._remove_all_images()
        if self.save_on_stop and not self.snap_on_start:
            _create_sparse_file(self._path_clean, self._size)
        return self

    @_coroutinized
    def verify(self):  # pylint: disable=invalid-overridden-method
        if self.snap_on_start:
            img = self.source._path_clean  # pylint: disable=protected-access
        elif self.save_on_stop:
            img = self._path_clean
        else:
            img = None

        if img is None or os.path.exists(img):
            return True
        raise qubes.storage.StoragePoolException(
            'Missing image file {!r} for volume {}'.format(img, self.vid))

    @qubes.storage.Volume.locked
    @_coroutinized
    def remove(self):  # pylint: disable=invalid-overridden-method
        self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
        self._remove_all_images()
        _remove_empty_dir(os.path.dirname(self._path_vid))
        return self

    def _remove_all_images(self):
        self._remove_incomplete_images()
        self._prune_revisions(keep=0)
        _remove_file(self._path_clean)
        _remove_file(self._path_dirty)

    def _remove_incomplete_images(self):
        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
            _remove_file(tmp)
        _remove_file(self._path_import)

    def is_outdated(self):
        if self.snap_on_start:
            with suppress(FileNotFoundError):
                # pylint: disable=protected-access
                return (os.path.getmtime(self.source._path_clean) >
                        os.path.getmtime(self._path_clean))
        return False

    def is_dirty(self):
        return self.save_on_stop and os.path.exists(self._path_dirty)

    @qubes.storage.Volume.locked
    @_coroutinized
    def start(self):  # pylint: disable=invalid-overridden-method
        self._remove_incomplete_images()
        if not self.is_dirty():
            if self.snap_on_start:
                # pylint: disable=protected-access
                _copy_file(self.source._path_clean, self._path_clean)
            if self.snap_on_start or self.save_on_stop:
                _copy_file(self._path_clean, self._path_dirty)
            else:
                # Preferably use the size of a leftover image, in case
                # the volume was previously resized - but then a crash
                # prevented qubes.xml serialization of the new size.
                _create_sparse_file(self._path_dirty, self.size)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def stop(self):  # pylint: disable=invalid-overridden-method
        if self.save_on_stop:
            if os.path.exists(self._path_dirty):
                self._commit(self._path_dirty)
        else:
            if not self.snap_on_start:
                self._size = self.size  # preserve manual resize of image
            _remove_file(self._path_dirty)
            _remove_file(self._path_clean)
        return self

    def _commit(self, path_from):
        self._add_revision()
        self._prune_revisions()
        qubes.utils.fsync_path(path_from)
        _rename_file(path_from, self._path_clean)

    def _add_revision(self):
        if self.revisions_to_keep == 0:
            return
        ctime = os.path.getctime(self._path_clean)
        timestamp = qubes.storage.isodate(int(ctime))
        _copy_file(self._path_clean,
                   self._path_revision(self._next_revision_number, timestamp))

    def _prune_revisions(self, keep=None):
        if keep is None:
            keep = self.revisions_to_keep
        for number, timestamp in list(self.revisions.items())[:-keep or None]:
            _remove_file(self._path_revision(number, timestamp))

    @qubes.storage.Volume.locked
    @_coroutinized
    def revert(self, revision=None):  # pylint: disable=invalid-overridden-method
        if self.is_dirty():
            raise qubes.storage.StoragePoolException(
                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
        if revision is None:
            number, timestamp = list(self.revisions.items())[-1]
        else:
            number, timestamp = revision, None
        path_revision = self._path_revision(number, timestamp)
        self._add_revision()
        _rename_file(path_revision, self._path_clean)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def resize(self, size):  # pylint: disable=invalid-overridden-method
        ''' Resize a read-write volume; notify any corresponding loop
            devices of the size change.
        '''
        if not self.rw:
            raise qubes.storage.StoragePoolException(
                'Cannot resize: {} is read-only'.format(self.vid))
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                _resize_file(path, size)
                break
        self._size = size
        if path == self._path_dirty:
            _update_loopdev_sizes(self._path_dirty)
        return self

    async def export(self):
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot export: {} is not save_on_stop'.format(self.vid))
        return self._path_clean

    @qubes.storage.Volume.locked
    @_coroutinized
    def import_data(self, size):  # pylint: disable=invalid-overridden-method
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
        _create_sparse_file(self._path_import, size)
        return self._path_import

    @_coroutinized
    def _import_data_end_unlocked(self, success):
        (self._commit if success else _remove_file)(self._path_import)
        return self

    import_data_end = qubes.storage.Volume.locked(_import_data_end_unlocked)

    @qubes.storage.Volume.locked
    async def import_volume(self, src_volume):
        if self.save_on_stop:
            try:
                success = False
                src_path = await qubes.utils.coro_maybe(src_volume.export())
                try:
                    await _coroutinized(_copy_file)(src_path, self._path_import)
                finally:
                    await qubes.utils.coro_maybe(
                        src_volume.export_end(src_path))
                success = True
            finally:
                await self._import_data_end_unlocked(success)
        return self

    def _path_revision(self, number, timestamp=None):
        if timestamp is None:
            timestamp = self.revisions[number]
        return self._path_clean + '.' + number + '@' + timestamp + 'Z'

    @property
    def _next_revision_number(self):
        numbers = self.revisions.keys()
        return str(int(list(numbers)[-1]) + 1) if numbers else '1'

    @property
    def revisions(self):
        prefix = self._path_clean + '.'
        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
        items = (path[len(prefix):-1].split('@') for path in paths)
        return collections.OrderedDict(
            sorted(items, key=lambda item: int(item[0])))

    @property
    def size(self):
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.path.getsize(path)
        return self._size

    @property
    def usage(self):
        ''' Return volume disk usage from the VM's perspective. It is
            usually much lower from the host's perspective due to CoW.
        '''
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.stat(path).st_blocks * 512
        return 0

def _replace_file(dst):
    _make_dir(os.path.dirname(dst))
    return qubes.utils.replace_file(
        dst, permissions=0o600, log_level=logging.INFO)

_rename_file = functools.partial(
    qubes.utils.rename_file, log_level=logging.INFO)

_remove_file = functools.partial(
    qubes.utils.remove_file, log_level=logging.INFO)

def _make_dir(path):
    ''' mkdir path, ignoring FileExistsError; return whether we
        created it.
    '''
    with suppress(FileExistsError):
        os.mkdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Created directory: %r', path)
        return True
    return False

def _remove_empty_dir(path):
    try:
        os.rmdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Removed empty directory: %r', path)
    except OSError as ex:
        if ex.errno not in (errno.ENOENT, errno.ENOTEMPTY):
            raise

def _resize_file(path, size):
    ''' Resize an existing file. '''
    with open(path, 'rb+') as file_io:
        file_io.truncate(size)
        os.fsync(file_io.fileno())

def _create_sparse_file(path, size):
    ''' Create an empty sparse file. '''

def _update_loopdev_sizes(img):
    ''' Resolve img; update the size of loop devices backed by it. '''
    needle = os.fsencode(os.path.realpath(img)) + b'\n'
    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
        matched = False
        with suppress(FileNotFoundError), open(sys_path, 'rb') as sys_io:
            matched = sys_io.read() == needle
        if matched:
            with open('/dev/' + sys_path.split('/')[3], 'rb') as dev_io:
                fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)

def _attempt_ficlone(src_io, dst_io):
    try:
        fcntl.ioctl(dst_io.fileno(), FICLONE, src_io.fileno())
        return True
    except OSError as ex:
        if ex.errno not in (errno.EBADF, errno.EINVAL,
                            errno.EOPNOTSUPP, errno.EXDEV):
            raise
        return False

def _copy_file(src, dst):
    ''' Copy src to dst as a reflink if possible, sparse if not. '''
    with _replace_file(dst) as tmp_io:
        with open(src, 'rb') as src_io:
            reflinked = _attempt_ficlone(src_io, tmp_io)
        if reflinked:
            LOGGER.info('Reflinked file: %r -> %r', src, tmp_io.name)
        else:
            LOGGER.info('Copying file: %r -> %r', src, tmp_io.name)
            result = subprocess.run(
                ['cp', '--sparse=always', '--', src, tmp_io.name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
            if result.returncode != 0:
                raise qubes.storage.StoragePoolException(str(result))
    return reflinked

def is_supported(dst_dir, src_dir=None):
    ''' Return whether destination directory supports reflink copies
        from source directory. (A temporary file is created in each
        directory, using O_TMPFILE if possible.)
    '''
    if src_dir is None:
        src_dir = dst_dir
    with tempfile.TemporaryFile(dir=src_dir) as src, \
         tempfile.TemporaryFile(dir=dst_dir) as dst:
        src.write(b'foo')  # don't let any fs get clever with empty files
        return _attempt_ficlone(src, dst)


if self.save_on_stop and not self.snap_on_start:
            _create_sparse_file(self._path_clean, self._size)

# The Qubes OS Project, http://example.com

''' Driver for handling VM images as files, without any device-mapper
    involvement. A reflink-capable filesystem is strongly recommended,
    but not required.
'''

import asyncio
import collections
import errno
import fcntl
import functools
import glob
import logging
import os
import platform
import subprocess
import tempfile
from contextlib import suppress

import qubes.storage
import qubes.utils

LOGGER = logging.getLogger('qubes.storage.reflink')

LOOP_SET_CAPACITY = 0x4C07

FICLONE = {
    'x86_64':  0x40049409,
    'ppc64le': 0x80049409,
}[platform.machine()]

def _coroutinized(function):
    ''' Wrap a synchronous function in a coroutine that runs the
        function via the event loop's ThreadPool-based default
        executor.
    '''
    @functools.wraps(function)
    async def wrapper(*args, **kwargs):
        return await asyncio.get_event_loop().run_in_executor(
            None, functools.partial(function, *args, **kwargs))
    return wrapper

class ReflinkPool(qubes.storage.Pool):
    driver = 'file-reflink'
    _known_dir_path_prefixes = ['appvms', 'vm-templates']

    def __init__(self, *, name, revisions_to_keep=1,
                 dir_path, setup_check=True, ephemeral_volatile=False):
        super().__init__(name=name, revisions_to_keep=revisions_to_keep,
                         ephemeral_volatile=ephemeral_volatile)
        self._setup_check = qubes.property.bool(None, None, setup_check)
        self._volumes = {}
        self.dir_path = os.path.abspath(dir_path)

    @_coroutinized
    def setup(self):  # pylint: disable=invalid-overridden-method
        created = _make_dir(self.dir_path)
        if self._setup_check and not is_supported(self.dir_path):
            if created:
                _remove_empty_dir(self.dir_path)
            raise qubes.storage.StoragePoolException(
                'The filesystem for {!r} does not support reflinks. If you'
                ' can live with VM startup delays and wasted disk space, pass'
                ' the "setup_check=False" option.'.format(self.dir_path))
        for dir_path_prefix in self._known_dir_path_prefixes:
            _make_dir(os.path.join(self.dir_path, dir_path_prefix))
        return self

    def init_volume(self, vm, volume_config):
        # Fail closed on any strange VM dir_path_prefix, just in case
        # /etc/udev/rules.d/00-qubes-ignore-devices.rules needs update
        assert vm.dir_path_prefix in self._known_dir_path_prefixes, \
               'Unknown dir_path_prefix {!r}'.format(vm.dir_path_prefix)

        volume_config['pool'] = self
        if 'revisions_to_keep' not in volume_config:
            volume_config['revisions_to_keep'] = self.revisions_to_keep
        if 'vid' not in volume_config:
            volume_config['vid'] = os.path.join(
                vm.dir_path_prefix, vm.name, volume_config['name'])
        volume = ReflinkVolume(**volume_config)
        self._volumes[volume_config['vid']] = volume
        return volume

    def list_volumes(self):
        return list(self._volumes.values())

    def get_volume(self, vid):
        return self._volumes[vid]

    async def destroy(self):
        pass

    @property
    def config(self):
        return {
            'name': self.name,
            'dir_path': self.dir_path,
            'driver': ReflinkPool.driver,
            'revisions_to_keep': self.revisions_to_keep,
            'ephemeral_volatile': self.ephemeral_volatile,
        }

    @property
    def size(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * statvfs.f_blocks

    @property
    def usage(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)

    def included_in(self, app):
        ''' Check if there is pool containing this one - either as a
        filesystem or its LVM volume'''
        return qubes.storage.search_pool_containing_dir(
            [pool for pool in app.pools.values() if pool is not self],
            self.dir_path)

class ReflinkVolume(qubes.storage.Volume):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
        self._path_clean = self._path_vid + '.img'
        self._path_dirty = self._path_vid + '-dirty.img'
        self._path_import = self._path_vid + '-import.img'
        self.path = self._path_dirty

    @qubes.storage.Volume.locked
    @_coroutinized
    def create(self):  # pylint: disable=invalid-overridden-method
        self._remove_all_images()

        return self

    @_coroutinized
    def verify(self):  # pylint: disable=invalid-overridden-method
        if self.snap_on_start:
            img = self.source._path_clean  # pylint: disable=protected-access
        elif self.save_on_stop:
            img = self._path_clean
        else:
            img = None

        if img is None or os.path.exists(img):
            return True
        raise qubes.storage.StoragePoolException(
            'Missing image file {!r} for volume {}'.format(img, self.vid))

    @qubes.storage.Volume.locked
    @_coroutinized
    def remove(self):  # pylint: disable=invalid-overridden-method
        self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
        self._remove_all_images()
        _remove_empty_dir(os.path.dirname(self._path_vid))
        return self

    def _remove_all_images(self):
        self._remove_incomplete_images()
        self._prune_revisions(keep=0)
        _remove_file(self._path_clean)
        _remove_file(self._path_dirty)

    def _remove_incomplete_images(self):
        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
            _remove_file(tmp)
        _remove_file(self._path_import)

    def is_outdated(self):
        if self.snap_on_start:
            with suppress(FileNotFoundError):
                # pylint: disable=protected-access
                return (os.path.getmtime(self.source._path_clean) >
                        os.path.getmtime(self._path_clean))
        return False

    def is_dirty(self):
        return self.save_on_stop and os.path.exists(self._path_dirty)

    @qubes.storage.Volume.locked
    @_coroutinized
    def start(self):  # pylint: disable=invalid-overridden-method
        self._remove_incomplete_images()
        if not self.is_dirty():
            if self.snap_on_start:
                # pylint: disable=protected-access
                _copy_file(self.source._path_clean, self._path_clean)
            if self.snap_on_start or self.save_on_stop:
                _copy_file(self._path_clean, self._path_dirty)
            else:
                # Preferably use the size of a leftover image, in case
                # the volume was previously resized - but then a crash
                # prevented qubes.xml serialization of the new size.
                _create_sparse_file(self._path_dirty, self.size)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def stop(self):  # pylint: disable=invalid-overridden-method
        if self.save_on_stop:
            if os.path.exists(self._path_dirty):
                self._commit(self._path_dirty)
        else:
            if not self.snap_on_start:
                self._size = self.size  # preserve manual resize of image
            _remove_file(self._path_dirty)
            _remove_file(self._path_clean)
        return self

    def _commit(self, path_from):
        self._add_revision()
        self._prune_revisions()
        qubes.utils.fsync_path(path_from)
        _rename_file(path_from, self._path_clean)

    def _add_revision(self):
        if self.revisions_to_keep == 0:
            return
        ctime = os.path.getctime(self._path_clean)
        timestamp = qubes.storage.isodate(int(ctime))
        _copy_file(self._path_clean,
                   self._path_revision(self._next_revision_number, timestamp))

    def _prune_revisions(self, keep=None):
        if keep is None:
            keep = self.revisions_to_keep
        for number, timestamp in list(self.revisions.items())[:-keep or None]:
            _remove_file(self._path_revision(number, timestamp))

    @qubes.storage.Volume.locked
    @_coroutinized
    def revert(self, revision=None):  # pylint: disable=invalid-overridden-method
        if self.is_dirty():
            raise qubes.storage.StoragePoolException(
                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
        if revision is None:
            number, timestamp = list(self.revisions.items())[-1]
        else:
            number, timestamp = revision, None
        path_revision = self._path_revision(number, timestamp)
        self._add_revision()
        _rename_file(path_revision, self._path_clean)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def resize(self, size):  # pylint: disable=invalid-overridden-method
        ''' Resize a read-write volume; notify any corresponding loop
            devices of the size change.
        '''
        if not self.rw:
            raise qubes.storage.StoragePoolException(
                'Cannot resize: {} is read-only'.format(self.vid))
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                _resize_file(path, size)
                break
        self._size = size
        if path == self._path_dirty:
            _update_loopdev_sizes(self._path_dirty)
        return self

    async def export(self):
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot export: {} is not save_on_stop'.format(self.vid))
        return self._path_clean

    @qubes.storage.Volume.locked
    @_coroutinized
    def import_data(self, size):  # pylint: disable=invalid-overridden-method
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
        _create_sparse_file(self._path_import, size)
        return self._path_import

    @_coroutinized
    def _import_data_end_unlocked(self, success):
        (self._commit if success else _remove_file)(self._path_import)
        return self

    import_data_end = qubes.storage.Volume.locked(_import_data_end_unlocked)

    @qubes.storage.Volume.locked
    async def import_volume(self, src_volume):
        if self.save_on_stop:
            try:
                success = False
                src_path = await qubes.utils.coro_maybe(src_volume.export())
                try:
                    await _coroutinized(_copy_file)(src_path, self._path_import)
                finally:
                    await qubes.utils.coro_maybe(
                        src_volume.export_end(src_path))
                success = True
            finally:
                await self._import_data_end_unlocked(success)
        return self

    def _path_revision(self, number, timestamp=None):
        if timestamp is None:
            timestamp = self.revisions[number]
        return self._path_clean + '.' + number + '@' + timestamp + 'Z'

    @property
    def _next_revision_number(self):
        numbers = self.revisions.keys()
        return str(int(list(numbers)[-1]) + 1) if numbers else '1'

    @property
    def revisions(self):
        prefix = self._path_clean + '.'
        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
        items = (path[len(prefix):-1].split('@') for path in paths)
        return collections.OrderedDict(
            sorted(items, key=lambda item: int(item[0])))

    @property
    def size(self):
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.path.getsize(path)
        return self._size

    @property
    def usage(self):
        ''' Return volume disk usage from the VM's perspective. It is
            usually much lower from the host's perspective due to CoW.
        '''
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.stat(path).st_blocks * 512
        return 0

def _replace_file(dst):
    _make_dir(os.path.dirname(dst))
    return qubes.utils.replace_file(
        dst, permissions=0o600, log_level=logging.INFO)

_rename_file = functools.partial(
    qubes.utils.rename_file, log_level=logging.INFO)

_remove_file = functools.partial(
    qubes.utils.remove_file, log_level=logging.INFO)

def _make_dir(path):
    ''' mkdir path, ignoring FileExistsError; return whether we
        created it.
    '''
    with suppress(FileExistsError):
        os.mkdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Created directory: %r', path)
        return True
    return False

def _remove_empty_dir(path):
    try:
        os.rmdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Removed empty directory: %r', path)
    except OSError as ex:
        if ex.errno not in (errno.ENOENT, errno.ENOTEMPTY):
            raise

def _resize_file(path, size):
    ''' Resize an existing file. '''
    with open(path, 'rb+') as file_io:
        file_io.truncate(size)
        os.fsync(file_io.fileno())

def _create_sparse_file(path, size):
    ''' Create an empty sparse file. '''
    with _replace_file(path) as tmp_io:
        tmp_io.truncate(size)
        LOGGER.info('Created sparse file: %r', tmp_io.name)

def _update_loopdev_sizes(img):
    ''' Resolve img; update the size of loop devices backed by it. '''
    needle = os.fsencode(os.path.realpath(img)) + b'\n'
    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
        matched = False
        with suppress(FileNotFoundError), open(sys_path, 'rb') as sys_io:
            matched = sys_io.read() == needle
        if matched:
            with open('/dev/' + sys_path.split('/')[3], 'rb') as dev_io:
                fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)

def _attempt_ficlone(src_io, dst_io):
    try:
        fcntl.ioctl(dst_io.fileno(), FICLONE, src_io.fileno())
        return True
    except OSError as ex:
        if ex.errno not in (errno.EBADF, errno.EINVAL,
                            errno.EOPNOTSUPP, errno.EXDEV):
            raise
        return False

def _copy_file(src, dst):
    ''' Copy src to dst as a reflink if possible, sparse if not. '''
    with _replace_file(dst) as tmp_io:
        with open(src, 'rb') as src_io:
            reflinked = _attempt_ficlone(src_io, tmp_io)
        if reflinked:
            LOGGER.info('Reflinked file: %r -> %r', src, tmp_io.name)
        else:
            LOGGER.info('Copying file: %r -> %r', src, tmp_io.name)
            result = subprocess.run(
                ['cp', '--sparse=always', '--', src, tmp_io.name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
            if result.returncode != 0:
                raise qubes.storage.StoragePoolException(str(result))
    return reflinked

def is_supported(dst_dir, src_dir=None):
    ''' Return whether destination directory supports reflink copies
        from source directory. (A temporary file is created in each
        directory, using O_TMPFILE if possible.)
    '''
    if src_dir is None:
        src_dir = dst_dir
    with tempfile.TemporaryFile(dir=src_dir) as src, \
         tempfile.TemporaryFile(dir=dst_dir) as dst:
        src.write(b'foo')  # don't let any fs get clever with empty files
        return _attempt_ficlone(src, dst)


for number, timestamp in list(self.revisions.items())[:-keep or None]:
            _remove_file(self._path_revision(number, timestamp))

# The Qubes OS Project, http://example.com

''' Driver for handling VM images as files, without any device-mapper
    involvement. A reflink-capable filesystem is strongly recommended,
    but not required.
'''

import asyncio
import collections
import errno
import fcntl
import functools
import glob
import logging
import os
import platform
import subprocess
import tempfile
from contextlib import suppress

import qubes.storage
import qubes.utils

LOGGER = logging.getLogger('qubes.storage.reflink')

LOOP_SET_CAPACITY = 0x4C07

FICLONE = {
    'x86_64':  0x40049409,
    'ppc64le': 0x80049409,
}[platform.machine()]

def _coroutinized(function):
    ''' Wrap a synchronous function in a coroutine that runs the
        function via the event loop's ThreadPool-based default
        executor.
    '''
    @functools.wraps(function)
    async def wrapper(*args, **kwargs):
        return await asyncio.get_event_loop().run_in_executor(
            None, functools.partial(function, *args, **kwargs))
    return wrapper

class ReflinkPool(qubes.storage.Pool):
    driver = 'file-reflink'
    _known_dir_path_prefixes = ['appvms', 'vm-templates']

    def __init__(self, *, name, revisions_to_keep=1,
                 dir_path, setup_check=True, ephemeral_volatile=False):
        super().__init__(name=name, revisions_to_keep=revisions_to_keep,
                         ephemeral_volatile=ephemeral_volatile)
        self._setup_check = qubes.property.bool(None, None, setup_check)
        self._volumes = {}
        self.dir_path = os.path.abspath(dir_path)

    @_coroutinized
    def setup(self):  # pylint: disable=invalid-overridden-method
        created = _make_dir(self.dir_path)
        if self._setup_check and not is_supported(self.dir_path):
            if created:
                _remove_empty_dir(self.dir_path)
            raise qubes.storage.StoragePoolException(
                'The filesystem for {!r} does not support reflinks. If you'
                ' can live with VM startup delays and wasted disk space, pass'
                ' the "setup_check=False" option.'.format(self.dir_path))
        for dir_path_prefix in self._known_dir_path_prefixes:
            _make_dir(os.path.join(self.dir_path, dir_path_prefix))
        return self

    def init_volume(self, vm, volume_config):
        # Fail closed on any strange VM dir_path_prefix, just in case
        # /etc/udev/rules.d/00-qubes-ignore-devices.rules needs update
        assert vm.dir_path_prefix in self._known_dir_path_prefixes, \
               'Unknown dir_path_prefix {!r}'.format(vm.dir_path_prefix)

        volume_config['pool'] = self
        if 'revisions_to_keep' not in volume_config:
            volume_config['revisions_to_keep'] = self.revisions_to_keep
        if 'vid' not in volume_config:
            volume_config['vid'] = os.path.join(
                vm.dir_path_prefix, vm.name, volume_config['name'])
        volume = ReflinkVolume(**volume_config)
        self._volumes[volume_config['vid']] = volume
        return volume

    def list_volumes(self):
        return list(self._volumes.values())

    def get_volume(self, vid):
        return self._volumes[vid]

    async def destroy(self):
        pass

    @property
    def config(self):
        return {
            'name': self.name,
            'dir_path': self.dir_path,
            'driver': ReflinkPool.driver,
            'revisions_to_keep': self.revisions_to_keep,
            'ephemeral_volatile': self.ephemeral_volatile,
        }

    @property
    def size(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * statvfs.f_blocks

    @property
    def usage(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)

    def included_in(self, app):
        ''' Check if there is pool containing this one - either as a
        filesystem or its LVM volume'''
        return qubes.storage.search_pool_containing_dir(
            [pool for pool in app.pools.values() if pool is not self],
            self.dir_path)

class ReflinkVolume(qubes.storage.Volume):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
        self._path_clean = self._path_vid + '.img'
        self._path_dirty = self._path_vid + '-dirty.img'
        self._path_import = self._path_vid + '-import.img'
        self.path = self._path_dirty

    @qubes.storage.Volume.locked
    @_coroutinized
    def create(self):  # pylint: disable=invalid-overridden-method
        self._remove_all_images()
        if self.save_on_stop and not self.snap_on_start:
            _create_sparse_file(self._path_clean, self._size)
        return self

    @_coroutinized
    def verify(self):  # pylint: disable=invalid-overridden-method
        if self.snap_on_start:
            img = self.source._path_clean  # pylint: disable=protected-access
        elif self.save_on_stop:
            img = self._path_clean
        else:
            img = None

        if img is None or os.path.exists(img):
            return True
        raise qubes.storage.StoragePoolException(
            'Missing image file {!r} for volume {}'.format(img, self.vid))

    @qubes.storage.Volume.locked
    @_coroutinized
    def remove(self):  # pylint: disable=invalid-overridden-method
        self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
        self._remove_all_images()
        _remove_empty_dir(os.path.dirname(self._path_vid))
        return self

    def _remove_all_images(self):
        self._remove_incomplete_images()
        self._prune_revisions(keep=0)
        _remove_file(self._path_clean)
        _remove_file(self._path_dirty)

    def _remove_incomplete_images(self):
        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
            _remove_file(tmp)
        _remove_file(self._path_import)

    def is_outdated(self):
        if self.snap_on_start:
            with suppress(FileNotFoundError):
                # pylint: disable=protected-access
                return (os.path.getmtime(self.source._path_clean) >
                        os.path.getmtime(self._path_clean))
        return False

    def is_dirty(self):
        return self.save_on_stop and os.path.exists(self._path_dirty)

    @qubes.storage.Volume.locked
    @_coroutinized
    def start(self):  # pylint: disable=invalid-overridden-method
        self._remove_incomplete_images()
        if not self.is_dirty():
            if self.snap_on_start:
                # pylint: disable=protected-access
                _copy_file(self.source._path_clean, self._path_clean)
            if self.snap_on_start or self.save_on_stop:
                _copy_file(self._path_clean, self._path_dirty)
            else:
                # Preferably use the size of a leftover image, in case
                # the volume was previously resized - but then a crash
                # prevented qubes.xml serialization of the new size.
                _create_sparse_file(self._path_dirty, self.size)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def stop(self):  # pylint: disable=invalid-overridden-method
        if self.save_on_stop:
            if os.path.exists(self._path_dirty):
                self._commit(self._path_dirty)
        else:
            if not self.snap_on_start:
                self._size = self.size  # preserve manual resize of image
            _remove_file(self._path_dirty)
            _remove_file(self._path_clean)
        return self

    def _commit(self, path_from):
        self._add_revision()
        self._prune_revisions()
        qubes.utils.fsync_path(path_from)
        _rename_file(path_from, self._path_clean)

    def _add_revision(self):
        if self.revisions_to_keep == 0:
            return
        ctime = os.path.getctime(self._path_clean)
        timestamp = qubes.storage.isodate(int(ctime))
        _copy_file(self._path_clean,
                   self._path_revision(self._next_revision_number, timestamp))

    def _prune_revisions(self, keep=None):
        if keep is None:
            keep = self.revisions_to_keep

    @qubes.storage.Volume.locked
    @_coroutinized
    def revert(self, revision=None):  # pylint: disable=invalid-overridden-method
        if self.is_dirty():
            raise qubes.storage.StoragePoolException(
                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
        if revision is None:
            number, timestamp = list(self.revisions.items())[-1]
        else:
            number, timestamp = revision, None
        path_revision = self._path_revision(number, timestamp)
        self._add_revision()
        _rename_file(path_revision, self._path_clean)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def resize(self, size):  # pylint: disable=invalid-overridden-method
        ''' Resize a read-write volume; notify any corresponding loop
            devices of the size change.
        '''
        if not self.rw:
            raise qubes.storage.StoragePoolException(
                'Cannot resize: {} is read-only'.format(self.vid))
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                _resize_file(path, size)
                break
        self._size = size
        if path == self._path_dirty:
            _update_loopdev_sizes(self._path_dirty)
        return self

    async def export(self):
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot export: {} is not save_on_stop'.format(self.vid))
        return self._path_clean

    @qubes.storage.Volume.locked
    @_coroutinized
    def import_data(self, size):  # pylint: disable=invalid-overridden-method
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
        _create_sparse_file(self._path_import, size)
        return self._path_import

    @_coroutinized
    def _import_data_end_unlocked(self, success):
        (self._commit if success else _remove_file)(self._path_import)
        return self

    import_data_end = qubes.storage.Volume.locked(_import_data_end_unlocked)

    @qubes.storage.Volume.locked
    async def import_volume(self, src_volume):
        if self.save_on_stop:
            try:
                success = False
                src_path = await qubes.utils.coro_maybe(src_volume.export())
                try:
                    await _coroutinized(_copy_file)(src_path, self._path_import)
                finally:
                    await qubes.utils.coro_maybe(
                        src_volume.export_end(src_path))
                success = True
            finally:
                await self._import_data_end_unlocked(success)
        return self

    def _path_revision(self, number, timestamp=None):
        if timestamp is None:
            timestamp = self.revisions[number]
        return self._path_clean + '.' + number + '@' + timestamp + 'Z'

    @property
    def _next_revision_number(self):
        numbers = self.revisions.keys()
        return str(int(list(numbers)[-1]) + 1) if numbers else '1'

    @property
    def revisions(self):
        prefix = self._path_clean + '.'
        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
        items = (path[len(prefix):-1].split('@') for path in paths)
        return collections.OrderedDict(
            sorted(items, key=lambda item: int(item[0])))

    @property
    def size(self):
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.path.getsize(path)
        return self._size

    @property
    def usage(self):
        ''' Return volume disk usage from the VM's perspective. It is
            usually much lower from the host's perspective due to CoW.
        '''
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.stat(path).st_blocks * 512
        return 0

def _replace_file(dst):
    _make_dir(os.path.dirname(dst))
    return qubes.utils.replace_file(
        dst, permissions=0o600, log_level=logging.INFO)

_rename_file = functools.partial(
    qubes.utils.rename_file, log_level=logging.INFO)

_remove_file = functools.partial(
    qubes.utils.remove_file, log_level=logging.INFO)

def _make_dir(path):
    ''' mkdir path, ignoring FileExistsError; return whether we
        created it.
    '''
    with suppress(FileExistsError):
        os.mkdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Created directory: %r', path)
        return True
    return False

def _remove_empty_dir(path):
    try:
        os.rmdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Removed empty directory: %r', path)
    except OSError as ex:
        if ex.errno not in (errno.ENOENT, errno.ENOTEMPTY):
            raise

def _resize_file(path, size):
    ''' Resize an existing file. '''
    with open(path, 'rb+') as file_io:
        file_io.truncate(size)
        os.fsync(file_io.fileno())

def _create_sparse_file(path, size):
    ''' Create an empty sparse file. '''
    with _replace_file(path) as tmp_io:
        tmp_io.truncate(size)
        LOGGER.info('Created sparse file: %r', tmp_io.name)

def _update_loopdev_sizes(img):
    ''' Resolve img; update the size of loop devices backed by it. '''
    needle = os.fsencode(os.path.realpath(img)) + b'\n'
    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
        matched = False
        with suppress(FileNotFoundError), open(sys_path, 'rb') as sys_io:
            matched = sys_io.read() == needle
        if matched:
            with open('/dev/' + sys_path.split('/')[3], 'rb') as dev_io:
                fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)

def _attempt_ficlone(src_io, dst_io):
    try:
        fcntl.ioctl(dst_io.fileno(), FICLONE, src_io.fileno())
        return True
    except OSError as ex:
        if ex.errno not in (errno.EBADF, errno.EINVAL,
                            errno.EOPNOTSUPP, errno.EXDEV):
            raise
        return False

def _copy_file(src, dst):
    ''' Copy src to dst as a reflink if possible, sparse if not. '''
    with _replace_file(dst) as tmp_io:
        with open(src, 'rb') as src_io:
            reflinked = _attempt_ficlone(src_io, tmp_io)
        if reflinked:
            LOGGER.info('Reflinked file: %r -> %r', src, tmp_io.name)
        else:
            LOGGER.info('Copying file: %r -> %r', src, tmp_io.name)
            result = subprocess.run(
                ['cp', '--sparse=always', '--', src, tmp_io.name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
            if result.returncode != 0:
                raise qubes.storage.StoragePoolException(str(result))
    return reflinked

def is_supported(dst_dir, src_dir=None):
    ''' Return whether destination directory supports reflink copies
        from source directory. (A temporary file is created in each
        directory, using O_TMPFILE if possible.)
    '''
    if src_dir is None:
        src_dir = dst_dir
    with tempfile.TemporaryFile(dir=src_dir) as src, \
         tempfile.TemporaryFile(dir=dst_dir) as dst:
        src.write(b'foo')  # don't let any fs get clever with empty files
        return _attempt_ficlone(src, dst)


try:
        fcntl.ioctl(dst_io.fileno(), FICLONE, src_io.fileno())
        return True
    except OSError as ex:
        if ex.errno not in (errno.EBADF, errno.EINVAL,
                            errno.EOPNOTSUPP, errno.EXDEV):
            raise
        return False

# The Qubes OS Project, http://example.com

''' Driver for handling VM images as files, without any device-mapper
    involvement. A reflink-capable filesystem is strongly recommended,
    but not required.
'''

import asyncio
import collections
import errno
import fcntl
import functools
import glob
import logging
import os
import platform
import subprocess
import tempfile
from contextlib import suppress

import qubes.storage
import qubes.utils

LOGGER = logging.getLogger('qubes.storage.reflink')

LOOP_SET_CAPACITY = 0x4C07

FICLONE = {
    'x86_64':  0x40049409,
    'ppc64le': 0x80049409,
}[platform.machine()]

def _coroutinized(function):
    ''' Wrap a synchronous function in a coroutine that runs the
        function via the event loop's ThreadPool-based default
        executor.
    '''
    @functools.wraps(function)
    async def wrapper(*args, **kwargs):
        return await asyncio.get_event_loop().run_in_executor(
            None, functools.partial(function, *args, **kwargs))
    return wrapper

class ReflinkPool(qubes.storage.Pool):
    driver = 'file-reflink'
    _known_dir_path_prefixes = ['appvms', 'vm-templates']

    def __init__(self, *, name, revisions_to_keep=1,
                 dir_path, setup_check=True, ephemeral_volatile=False):
        super().__init__(name=name, revisions_to_keep=revisions_to_keep,
                         ephemeral_volatile=ephemeral_volatile)
        self._setup_check = qubes.property.bool(None, None, setup_check)
        self._volumes = {}
        self.dir_path = os.path.abspath(dir_path)

    @_coroutinized
    def setup(self):  # pylint: disable=invalid-overridden-method
        created = _make_dir(self.dir_path)
        if self._setup_check and not is_supported(self.dir_path):
            if created:
                _remove_empty_dir(self.dir_path)
            raise qubes.storage.StoragePoolException(
                'The filesystem for {!r} does not support reflinks. If you'
                ' can live with VM startup delays and wasted disk space, pass'
                ' the "setup_check=False" option.'.format(self.dir_path))
        for dir_path_prefix in self._known_dir_path_prefixes:
            _make_dir(os.path.join(self.dir_path, dir_path_prefix))
        return self

    def init_volume(self, vm, volume_config):
        # Fail closed on any strange VM dir_path_prefix, just in case
        # /etc/udev/rules.d/00-qubes-ignore-devices.rules needs update
        assert vm.dir_path_prefix in self._known_dir_path_prefixes, \
               'Unknown dir_path_prefix {!r}'.format(vm.dir_path_prefix)

        volume_config['pool'] = self
        if 'revisions_to_keep' not in volume_config:
            volume_config['revisions_to_keep'] = self.revisions_to_keep
        if 'vid' not in volume_config:
            volume_config['vid'] = os.path.join(
                vm.dir_path_prefix, vm.name, volume_config['name'])
        volume = ReflinkVolume(**volume_config)
        self._volumes[volume_config['vid']] = volume
        return volume

    def list_volumes(self):
        return list(self._volumes.values())

    def get_volume(self, vid):
        return self._volumes[vid]

    async def destroy(self):
        pass

    @property
    def config(self):
        return {
            'name': self.name,
            'dir_path': self.dir_path,
            'driver': ReflinkPool.driver,
            'revisions_to_keep': self.revisions_to_keep,
            'ephemeral_volatile': self.ephemeral_volatile,
        }

    @property
    def size(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * statvfs.f_blocks

    @property
    def usage(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)

    def included_in(self, app):
        ''' Check if there is pool containing this one - either as a
        filesystem or its LVM volume'''
        return qubes.storage.search_pool_containing_dir(
            [pool for pool in app.pools.values() if pool is not self],
            self.dir_path)

class ReflinkVolume(qubes.storage.Volume):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
        self._path_clean = self._path_vid + '.img'
        self._path_dirty = self._path_vid + '-dirty.img'
        self._path_import = self._path_vid + '-import.img'
        self.path = self._path_dirty

    @qubes.storage.Volume.locked
    @_coroutinized
    def create(self):  # pylint: disable=invalid-overridden-method
        self._remove_all_images()
        if self.save_on_stop and not self.snap_on_start:
            _create_sparse_file(self._path_clean, self._size)
        return self

    @_coroutinized
    def verify(self):  # pylint: disable=invalid-overridden-method
        if self.snap_on_start:
            img = self.source._path_clean  # pylint: disable=protected-access
        elif self.save_on_stop:
            img = self._path_clean
        else:
            img = None

        if img is None or os.path.exists(img):
            return True
        raise qubes.storage.StoragePoolException(
            'Missing image file {!r} for volume {}'.format(img, self.vid))

    @qubes.storage.Volume.locked
    @_coroutinized
    def remove(self):  # pylint: disable=invalid-overridden-method
        self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
        self._remove_all_images()
        _remove_empty_dir(os.path.dirname(self._path_vid))
        return self

    def _remove_all_images(self):
        self._remove_incomplete_images()
        self._prune_revisions(keep=0)
        _remove_file(self._path_clean)
        _remove_file(self._path_dirty)

    def _remove_incomplete_images(self):
        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
            _remove_file(tmp)
        _remove_file(self._path_import)

    def is_outdated(self):
        if self.snap_on_start:
            with suppress(FileNotFoundError):
                # pylint: disable=protected-access
                return (os.path.getmtime(self.source._path_clean) >
                        os.path.getmtime(self._path_clean))
        return False

    def is_dirty(self):
        return self.save_on_stop and os.path.exists(self._path_dirty)

    @qubes.storage.Volume.locked
    @_coroutinized
    def start(self):  # pylint: disable=invalid-overridden-method
        self._remove_incomplete_images()
        if not self.is_dirty():
            if self.snap_on_start:
                # pylint: disable=protected-access
                _copy_file(self.source._path_clean, self._path_clean)
            if self.snap_on_start or self.save_on_stop:
                _copy_file(self._path_clean, self._path_dirty)
            else:
                # Preferably use the size of a leftover image, in case
                # the volume was previously resized - but then a crash
                # prevented qubes.xml serialization of the new size.
                _create_sparse_file(self._path_dirty, self.size)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def stop(self):  # pylint: disable=invalid-overridden-method
        if self.save_on_stop:
            if os.path.exists(self._path_dirty):
                self._commit(self._path_dirty)
        else:
            if not self.snap_on_start:
                self._size = self.size  # preserve manual resize of image
            _remove_file(self._path_dirty)
            _remove_file(self._path_clean)
        return self

    def _commit(self, path_from):
        self._add_revision()
        self._prune_revisions()
        qubes.utils.fsync_path(path_from)
        _rename_file(path_from, self._path_clean)

    def _add_revision(self):
        if self.revisions_to_keep == 0:
            return
        ctime = os.path.getctime(self._path_clean)
        timestamp = qubes.storage.isodate(int(ctime))
        _copy_file(self._path_clean,
                   self._path_revision(self._next_revision_number, timestamp))

    def _prune_revisions(self, keep=None):
        if keep is None:
            keep = self.revisions_to_keep
        for number, timestamp in list(self.revisions.items())[:-keep or None]:
            _remove_file(self._path_revision(number, timestamp))

    @qubes.storage.Volume.locked
    @_coroutinized
    def revert(self, revision=None):  # pylint: disable=invalid-overridden-method
        if self.is_dirty():
            raise qubes.storage.StoragePoolException(
                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
        if revision is None:
            number, timestamp = list(self.revisions.items())[-1]
        else:
            number, timestamp = revision, None
        path_revision = self._path_revision(number, timestamp)
        self._add_revision()
        _rename_file(path_revision, self._path_clean)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def resize(self, size):  # pylint: disable=invalid-overridden-method
        ''' Resize a read-write volume; notify any corresponding loop
            devices of the size change.
        '''
        if not self.rw:
            raise qubes.storage.StoragePoolException(
                'Cannot resize: {} is read-only'.format(self.vid))
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                _resize_file(path, size)
                break
        self._size = size
        if path == self._path_dirty:
            _update_loopdev_sizes(self._path_dirty)
        return self

    async def export(self):
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot export: {} is not save_on_stop'.format(self.vid))
        return self._path_clean

    @qubes.storage.Volume.locked
    @_coroutinized
    def import_data(self, size):  # pylint: disable=invalid-overridden-method
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
        _create_sparse_file(self._path_import, size)
        return self._path_import

    @_coroutinized
    def _import_data_end_unlocked(self, success):
        (self._commit if success else _remove_file)(self._path_import)
        return self

    import_data_end = qubes.storage.Volume.locked(_import_data_end_unlocked)

    @qubes.storage.Volume.locked
    async def import_volume(self, src_volume):
        if self.save_on_stop:
            try:
                success = False
                src_path = await qubes.utils.coro_maybe(src_volume.export())
                try:
                    await _coroutinized(_copy_file)(src_path, self._path_import)
                finally:
                    await qubes.utils.coro_maybe(
                        src_volume.export_end(src_path))
                success = True
            finally:
                await self._import_data_end_unlocked(success)
        return self

    def _path_revision(self, number, timestamp=None):
        if timestamp is None:
            timestamp = self.revisions[number]
        return self._path_clean + '.' + number + '@' + timestamp + 'Z'

    @property
    def _next_revision_number(self):
        numbers = self.revisions.keys()
        return str(int(list(numbers)[-1]) + 1) if numbers else '1'

    @property
    def revisions(self):
        prefix = self._path_clean + '.'
        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
        items = (path[len(prefix):-1].split('@') for path in paths)
        return collections.OrderedDict(
            sorted(items, key=lambda item: int(item[0])))

    @property
    def size(self):
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.path.getsize(path)
        return self._size

    @property
    def usage(self):
        ''' Return volume disk usage from the VM's perspective. It is
            usually much lower from the host's perspective due to CoW.
        '''
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.stat(path).st_blocks * 512
        return 0

def _replace_file(dst):
    _make_dir(os.path.dirname(dst))
    return qubes.utils.replace_file(
        dst, permissions=0o600, log_level=logging.INFO)

_rename_file = functools.partial(
    qubes.utils.rename_file, log_level=logging.INFO)

_remove_file = functools.partial(
    qubes.utils.remove_file, log_level=logging.INFO)

def _make_dir(path):
    ''' mkdir path, ignoring FileExistsError; return whether we
        created it.
    '''
    with suppress(FileExistsError):
        os.mkdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Created directory: %r', path)
        return True
    return False

def _remove_empty_dir(path):
    try:
        os.rmdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Removed empty directory: %r', path)
    except OSError as ex:
        if ex.errno not in (errno.ENOENT, errno.ENOTEMPTY):
            raise

def _resize_file(path, size):
    ''' Resize an existing file. '''
    with open(path, 'rb+') as file_io:
        file_io.truncate(size)
        os.fsync(file_io.fileno())

def _create_sparse_file(path, size):
    ''' Create an empty sparse file. '''
    with _replace_file(path) as tmp_io:
        tmp_io.truncate(size)
        LOGGER.info('Created sparse file: %r', tmp_io.name)

def _update_loopdev_sizes(img):
    ''' Resolve img; update the size of loop devices backed by it. '''
    needle = os.fsencode(os.path.realpath(img)) + b'\n'
    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
        matched = False
        with suppress(FileNotFoundError), open(sys_path, 'rb') as sys_io:
            matched = sys_io.read() == needle
        if matched:
            with open('/dev/' + sys_path.split('/')[3], 'rb') as dev_io:
                fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)

def _attempt_ficlone(src_io, dst_io):

def _copy_file(src, dst):
    ''' Copy src to dst as a reflink if possible, sparse if not. '''
    with _replace_file(dst) as tmp_io:
        with open(src, 'rb') as src_io:
            reflinked = _attempt_ficlone(src_io, tmp_io)
        if reflinked:
            LOGGER.info('Reflinked file: %r -> %r', src, tmp_io.name)
        else:
            LOGGER.info('Copying file: %r -> %r', src, tmp_io.name)
            result = subprocess.run(
                ['cp', '--sparse=always', '--', src, tmp_io.name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
            if result.returncode != 0:
                raise qubes.storage.StoragePoolException(str(result))
    return reflinked

def is_supported(dst_dir, src_dir=None):
    ''' Return whether destination directory supports reflink copies
        from source directory. (A temporary file is created in each
        directory, using O_TMPFILE if possible.)
    '''
    if src_dir is None:
        src_dir = dst_dir
    with tempfile.TemporaryFile(dir=src_dir) as src, \
         tempfile.TemporaryFile(dir=dst_dir) as dst:
        src.write(b'foo')  # don't let any fs get clever with empty files
        return _attempt_ficlone(src, dst)


# The Qubes OS Project, http://example.com

''' Driver for handling VM images as files, without any device-mapper
    involvement. A reflink-capable filesystem is strongly recommended,
    but not required.
'''

import asyncio
import collections
import errno
import fcntl
import functools
import glob
import logging
import os
import platform
import subprocess
import tempfile
from contextlib import suppress

import qubes.storage
import qubes.utils

LOGGER = logging.getLogger('qubes.storage.reflink')

LOOP_SET_CAPACITY = 0x4C07

FICLONE = {
    'x86_64':  0x40049409,
    'ppc64le': 0x80049409,
}[platform.machine()]

def _coroutinized(function):
    ''' Wrap a synchronous function in a coroutine that runs the
        function via the event loop's ThreadPool-based default
        executor.
    '''
    @functools.wraps(function)
    async def wrapper(*args, **kwargs):
        return await asyncio.get_event_loop().run_in_executor(
            None, functools.partial(function, *args, **kwargs))
    return wrapper

class ReflinkPool(qubes.storage.Pool):
    driver = 'file-reflink'
    _known_dir_path_prefixes = ['appvms', 'vm-templates']

    def __init__(self, *, name, revisions_to_keep=1,
                 dir_path, setup_check=True, ephemeral_volatile=False):
        super().__init__(name=name, revisions_to_keep=revisions_to_keep,
                         ephemeral_volatile=ephemeral_volatile)
        self._setup_check = qubes.property.bool(None, None, setup_check)
        self._volumes = {}
        self.dir_path = os.path.abspath(dir_path)

    @_coroutinized
    def setup(self):  # pylint: disable=invalid-overridden-method
        created = _make_dir(self.dir_path)
        if self._setup_check and not is_supported(self.dir_path):
            if created:
                _remove_empty_dir(self.dir_path)
            raise qubes.storage.StoragePoolException(
                'The filesystem for {!r} does not support reflinks. If you'
                ' can live with VM startup delays and wasted disk space, pass'
                ' the "setup_check=False" option.'.format(self.dir_path))
        for dir_path_prefix in self._known_dir_path_prefixes:
            _make_dir(os.path.join(self.dir_path, dir_path_prefix))
        return self

    def init_volume(self, vm, volume_config):
        # Fail closed on any strange VM dir_path_prefix, just in case
        # /etc/udev/rules.d/00-qubes-ignore-devices.rules needs update
        assert vm.dir_path_prefix in self._known_dir_path_prefixes, \
               'Unknown dir_path_prefix {!r}'.format(vm.dir_path_prefix)

        volume_config['pool'] = self
        if 'revisions_to_keep' not in volume_config:
            volume_config['revisions_to_keep'] = self.revisions_to_keep
        if 'vid' not in volume_config:
            volume_config['vid'] = os.path.join(
                vm.dir_path_prefix, vm.name, volume_config['name'])
        volume = ReflinkVolume(**volume_config)

     self._volumes[volume_config['vid']] = volume
        return volume

    def list_volumes(self):
        return list(self._volumes.values())

    def get_volume(self, vid):
        return self._volumes[vid]

    async def destroy(self):
        pass

    @property
    def config(self):
        return {
            'name': self.name,
            'dir_path': self.dir_path,
            'driver': ReflinkPool.driver,
            'revisions_to_keep': self.revisions_to_keep,
            'ephemeral_volatile': self.ephemeral_volatile,
        }

    @property
    def size(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * statvfs.f_blocks

    @property
    def usage(self):
        statvfs = os.statvfs(self.dir_path)
        return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)

    def included_in(self, app):
        ''' Check if there is pool containing this one - either as a
        filesystem or its LVM volume'''
        return qubes.storage.search_pool_containing_dir(
            [pool for pool in app.pools.values() if pool is not self],
            self.dir_path)

class ReflinkVolume(qubes.storage.Volume):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
        self._path_clean = self._path_vid + '.img'
        self._path_dirty = self._path_vid + '-dirty.img'
        self._path_import = self._path_vid + '-import.img'
        self.path = self._path_dirty

    @qubes.storage.Volume.locked
    @_coroutinized
    def create(self):  # pylint: disable=invalid-overridden-method
        self._remove_all_images()
        if self.save_on_stop and not self.snap_on_start:
            _create_sparse_file(self._path_clean, self._size)
        return self

    @_coroutinized
    def verify(self):  # pylint: disable=invalid-overridden-method
        if self.snap_on_start:
            img = self.source._path_clean  # pylint: disable=protected-access
        elif self.save_on_stop:
            img = self._path_clean
        else:
            img = None

        if img is None or os.path.exists(img):
            return True
        raise qubes.storage.StoragePoolException(
            'Missing image file {!r} for volume {}'.format(img, self.vid))

    @qubes.storage.Volume.locked
    @_coroutinized
    def remove(self):  # pylint: disable=invalid-overridden-method
        self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
        self._remove_all_images()
        _remove_empty_dir(os.path.dirname(self._path_vid))
        return self

    def _remove_all_images(self):
        self._remove_incomplete_images()
        self._prune_revisions(keep=0)
        _remove_file(self._path_clean)
        _remove_file(self._path_dirty)

    def _remove_incomplete_images(self):
        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
            _remove_file(tmp)
        _remove_file(self._path_import)

    def is_outdated(self):
        if self.snap_on_start:
            with suppress(FileNotFoundError):
                # pylint: disable=protected-access
                return (os.path.getmtime(self.source._path_clean) >
                        os.path.getmtime(self._path_clean))
        return False

    def is_dirty(self):
        return self.save_on_stop and os.path.exists(self._path_dirty)

    @qubes.storage.Volume.locked
    @_coroutinized
    def start(self):  # pylint: disable=invalid-overridden-method
        self._remove_incomplete_images()
        if not self.is_dirty():
            if self.snap_on_start:
                # pylint: disable=protected-access
                _copy_file(self.source._path_clean, self._path_clean)
            if self.snap_on_start or self.save_on_stop:
                _copy_file(self._path_clean, self._path_dirty)
            else:
                # Preferably use the size of a leftover image, in case
                # the volume was previously resized - but then a crash
                # prevented qubes.xml serialization of the new size.
                _create_sparse_file(self._path_dirty, self.size)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def stop(self):  # pylint: disable=invalid-overridden-method
        if self.save_on_stop:
            if os.path.exists(self._path_dirty):
                self._commit(self._path_dirty)
        else:
            if not self.snap_on_start:
                self._size = self.size  # preserve manual resize of image
            _remove_file(self._path_dirty)
            _remove_file(self._path_clean)
        return self

    def _commit(self, path_from):
        self._add_revision()
        self._prune_revisions()
        qubes.utils.fsync_path(path_from)
        _rename_file(path_from, self._path_clean)

    def _add_revision(self):
        if self.revisions_to_keep == 0:
            return
        ctime = os.path.getctime(self._path_clean)
        timestamp = qubes.storage.isodate(int(ctime))
        _copy_file(self._path_clean,
                   self._path_revision(self._next_revision_number, timestamp))

    def _prune_revisions(self, keep=None):
        if keep is None:
            keep = self.revisions_to_keep
        for number, timestamp in list(self.revisions.items())[:-keep or None]:
            _remove_file(self._path_revision(number, timestamp))

    @qubes.storage.Volume.locked
    @_coroutinized
    def revert(self, revision=None):  # pylint: disable=invalid-overridden-method
        if self.is_dirty():
            raise qubes.storage.StoragePoolException(
                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
        if revision is None:
            number, timestamp = list(self.revisions.items())[-1]
        else:
            number, timestamp = revision, None
        path_revision = self._path_revision(number, timestamp)
        self._add_revision()
        _rename_file(path_revision, self._path_clean)
        return self

    @qubes.storage.Volume.locked
    @_coroutinized
    def resize(self, size):  # pylint: disable=invalid-overridden-method
        ''' Resize a read-write volume; notify any corresponding loop
            devices of the size change.
        '''
        if not self.rw:
            raise qubes.storage.StoragePoolException(
                'Cannot resize: {} is read-only'.format(self.vid))
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                _resize_file(path, size)
                break
        self._size = size
        if path == self._path_dirty:
            _update_loopdev_sizes(self._path_dirty)
        return self

    async def export(self):
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot export: {} is not save_on_stop'.format(self.vid))
        return self._path_clean

    @qubes.storage.Volume.locked
    @_coroutinized
    def import_data(self, size):  # pylint: disable=invalid-overridden-method
        if not self.save_on_stop:
            raise NotImplementedError(
                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
        _create_sparse_file(self._path_import, size)
        return self._path_import

    @_coroutinized
    def _import_data_end_unlocked(self, success):
        (self._commit if success else _remove_file)(self._path_import)
        return self

    import_data_end = qubes.storage.Volume.locked(_import_data_end_unlocked)

    @qubes.storage.Volume.locked
    async def import_volume(self, src_volume):
        if self.save_on_stop:
            try:
                success = False
                src_path = await qubes.utils.coro_maybe(src_volume.export())
                try:
                    await _coroutinized(_copy_file)(src_path, self._path_import)
                finally:
                    await qubes.utils.coro_maybe(
                        src_volume.export_end(src_path))
                success = True
            finally:
                await self._import_data_end_unlocked(success)
        return self

    def _path_revision(self, number, timestamp=None):
        if timestamp is None:
            timestamp = self.revisions[number]
        return self._path_clean + '.' + number + '@' + timestamp + 'Z'

    @property
    def _next_revision_number(self):
        numbers = self.revisions.keys()
        return str(int(list(numbers)[-1]) + 1) if numbers else '1'

    @property
    def revisions(self):
        prefix = self._path_clean + '.'
        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
        items = (path[len(prefix):-1].split('@') for path in paths)
        return collections.OrderedDict(
            sorted(items, key=lambda item: int(item[0])))

    @property
    def size(self):
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.path.getsize(path)
        return self._size

    @property
    def usage(self):
        ''' Return volume disk usage from the VM's perspective. It is
            usually much lower from the host's perspective due to CoW.
        '''
        for path in (self._path_dirty, self._path_clean):
            with suppress(FileNotFoundError):
                return os.stat(path).st_blocks * 512
        return 0

def _replace_file(dst):
    _make_dir(os.path.dirname(dst))
    return qubes.utils.replace_file(
        dst, permissions=0o600, log_level=logging.INFO)

_rename_file = functools.partial(
    qubes.utils.rename_file, log_level=logging.INFO)

_remove_file = functools.partial(
    qubes.utils.remove_file, log_level=logging.INFO)

def _make_dir(path):
    ''' mkdir path, ignoring FileExistsError; return whether we
        created it.
    '''
    with suppress(FileExistsError):
        os.mkdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Created directory: %r', path)
        return True
    return False

def _remove_empty_dir(path):
    try:
        os.rmdir(path)
        qubes.utils.fsync_path(os.path.dirname(path))
        LOGGER.info('Removed empty directory: %r', path)
    except OSError as ex:
        if ex.errno not in (errno.ENOENT, errno.ENOTEMPTY):
            raise

def _resize_file(path, size):
    ''' Resize an existing file. '''
    with open(path, 'rb+') as file_io:
        file_io.truncate(size)
        os.fsync(file_io.fileno())

def _create_sparse_file(path, size):
    ''' Create an empty sparse file. '''
    with _replace_file(path) as tmp_io:
        tmp_io.truncate(size)
        LOGGER.info('Created sparse file: %r', tmp_io.name)

def _update_loopdev_sizes(img):
    ''' Resolve img; update the size of loop devices backed by it. '''
    needle = os.fsencode(os.path.realpath(img)) + b'\n'
    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
        matched = False
        with suppress(FileNotFoundError), open(sys_path, 'rb') as sys_io:
            matched = sys_io.read() == needle
        if matched:
            with open('/dev/' + sys_path.split('/')[3], 'rb') as dev_io:
                fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)

def _attempt_ficlone(src_io, dst_io):
    try:
        fcntl.ioctl(dst_io.fileno(), FICLONE, src_io.fileno())
        return True
    except OSError as ex:
        if ex.errno not in (errno.EBADF, errno.EINVAL,
                            errno.EOPNOTSUPP, errno.EXDEV):
            raise
        return False

def _copy_file(src, dst):
    ''' Copy src to dst as a reflink if possible, sparse if not. '''
    with _replace_file(dst) as tmp_io:
        with open(src, 'rb') as src_io:
            reflinked = _attempt_ficlone(src_io, tmp_io)
        if reflinked:
            LOGGER.info('Reflinked file: %r -> %r', src, tmp_io.name)
        else:
            LOGGER.info('Copying file: %r -> %r', src, tmp_io.name)
            result = subprocess.run(
                ['cp', '--sparse=always', '--', src, tmp_io.name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
            if result.returncode != 0:
                raise qubes.storage.StoragePoolException(str(result))
    return reflinked

def is_supported(dst_dir, src_dir=None):
    ''' Return whether destination directory supports reflink copies
        from source directory. (A temporary file is created in each
        directory, using O_TMPFILE if possible.)
    '''
    if src_dir is None:
        src_dir = dst_dir
    with tempfile.TemporaryFile(dir=src_dir) as src, \
         tempfile.TemporaryFile(dir=dst_dir) as dst:
        src.write(b'foo')  # don't let any fs get clever with empty files
        return _attempt_ficlone(src, dst)


import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

""" 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


""" 关闭数据库 """
        self.client.close()

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


""" 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):

        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


item_list = data['listItem']
        if len(item_list) == 0:
            return []

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []

        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.item

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """


if len(item_list) == 0:
            return []

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']


for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


try:
            data = json.loads(body)
        except:
            return []

import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:
            print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []


import json
import sys
sys.path.append("../")
from lib.model import *
from lib.utils import *
from lib.config import *

reload(sys)
sys.setdefaultencoding('utf8')

class ItemCrawler:
    """ 爬取淘宝文胸商品记录 """

    def __init__(self):
        self.client = init_client()
        self.db = self.client[config['db_name']]
        self.collection = self.db.items

    def run(self):
        urls = []
        for i in range(1, 100 + 1):
            urls.append("http://s.m.taobao.com/search?q=文胸&m=api4h5&page=" + str(i))
            urls.append("http://s.m.taobao.com/search?q=胸罩&m=api4h5&page=" + str(i))
            urls.append("http://example.com=" + str(i))

        for url in urls:

print url
            body = get_body(url)
            if len(body) == 0:
                continue
            items = self.__parse(body)
            self.__add_items(items)

        self.__close()

    def __parse(self, body):
        """ 解析商品记录 """
        items = []
        try:
            data = json.loads(body)
        except:
            return []
        item_list = data['listItem']
        if len(item_list) == 0:
            return []
        for _item in item_list:
            item = Item(_item['item_id'], _item['userId'], _item['title'], False)
            items.append(item)
        return items

    def __add_items(self, items):
        """ 添加商品记录到数据库 """
        for item in items:
            if self.collection.find({'item_id': item.item_id}).count() == 0:
                self.collection.insert(item.dict())

    def __close(self):
        """ 关闭数据库 """
        self.client.close()

if __name__ == '__main__':
    crawler = ItemCrawler()
    crawler.run()


import pytest  # noqa
from parglare import Grammar, Parser
from parglare.actions import pass_single

grammar = r"""
E: E '+' E  {left}
 | number;

terminals
number: /\d+(\.\d+)?/;
"""

called = [False, False]

def act_sum(is_tree):
    def act_sum(context, nodes):
        called[0] = True
        assert context.parser
        assert context.state.symbol.name == 'E'
        assert context.production.symbol.name == 'E'
        assert len(context.production.rhs) == 3
        assert context.layout_content == '   '
        assert context.start_position == 3
        assert context.end_position == 8
        if is_tree:
            # If parse tree is constructed `node` is available on
            # the context.
            assert context.node.is_nonterm() \
                and context.node.symbol.name == 'E'
        else:
            context.node is None

    return act_sum

def act_number(context, value):
    called[1] = True
    value = float(value)
    assert context.symbol.name == 'number'
    if value == 1:
        assert context.start_position == 3
        assert context.end_position == 4
        assert context.layout_content == '   '
    else:
        assert context.start_position == 7
        assert context.end_position == 8
        assert context.layout_content == ' '
    return value

actions = {
    "Result": pass_single,
    "E": [None, pass_single],
    "number": act_number,
}

g = Grammar.from_string(grammar)

def test_parse_context():
    global called
    called = [False, False]

    actions["E"][0] = act_sum(is_tree=False)
    parser = Parser(g, actions=actions)

    parser.parse("   1 + 2  ")

    assert all(called)

"""
    Test that valid context attributes are available when calling
    actions using `call_actions`.
    """
    global called
    called = [False, False]

    actions["E"][0] = act_sum(is_tree=True)
    parser = Parser(g, build_tree=True, actions=actions)

    tree = parser.parse("   1 + 2  ")

    parser.call_actions(tree)

    assert all(called)


called[1] = True
    value = float(value)
    assert context.symbol.name == 'number'
    if value == 1:
        assert context.start_position == 3
        assert context.end_position == 4
        assert context.layout_content == '   '
    else:
        assert context.start_position == 7
        assert context.end_position == 8
        assert context.layout_content == ' '
    return value

import pytest  # noqa
from parglare import Grammar, Parser
from parglare.actions import pass_single

grammar = r"""
E: E '+' E  {left}
 | number;

terminals
number: /\d+(\.\d+)?/;
"""

called = [False, False]

def act_sum(is_tree):
    def act_sum(context, nodes):
        called[0] = True
        assert context.parser
        assert context.state.symbol.name == 'E'
        assert context.production.symbol.name == 'E'
        assert len(context.production.rhs) == 3
        assert context.layout_content == '   '
        assert context.start_position == 3
        assert context.end_position == 8
        if is_tree:
            # If parse tree is constructed `node` is available on
            # the context.
            assert context.node.is_nonterm() \
                and context.node.symbol.name == 'E'
        else:
            context.node is None

    return act_sum

def act_number(context, value):

actions = {
    "Result": pass_single,
    "E": [None, pass_single],
    "number": act_number,
}

g = Grammar.from_string(grammar)

def test_parse_context():
    global called
    called = [False, False]

    actions["E"][0] = act_sum(is_tree=False)
    parser = Parser(g, actions=actions)

    parser.parse("   1 + 2  ")

    assert all(called)

def test_parse_context_call_actions():
    """
    Test that valid context attributes are available when calling
    actions using `call_actions`.
    """
    global called
    called = [False, False]

    actions["E"][0] = act_sum(is_tree=True)
    parser = Parser(g, build_tree=True, actions=actions)

    tree = parser.parse("   1 + 2  ")

    parser.call_actions(tree)

    assert all(called)