if len(fileobjs) != 1:
        raise NotImplementedError('Cannot currently sniff hints from multiple '
                                  'files--please provide hints')
    fileobj = fileobjs[0]
    hints = sniff_hints(fileobj, initial_hints=initial_hints)
    return hints

import chardet
from contextlib import contextmanager
from . import PartialRecordsHints
from .csv_streamer import stream_csv, python_encoding_from_hint
import io
import csv
import gzip
import bz2
from .types import HintEncoding, HintRecordTerminator, HintQuoting, HintCompression
from .conversions import hint_encoding_from_chardet
from typing import List, IO, Optional, Iterator, Dict
from records_mover.utils.rewound_fileobj import rewound_fileobj
from records_mover.mover_types import _assert_never
import logging

logger = logging.getLogger(__name__)

HINT_INFERENCE_SAMPLING_SIZE_BYTES = 1024

@contextmanager
def rewound_decompressed_fileobj(fileobj: IO[bytes],
                                 compression: HintCompression) -> Iterator[IO[bytes]]:
    with rewound_fileobj(fileobj) as fileobj_after_rewind:
        if compression is None:
            yield fileobj
        elif compression == 'GZIP':
            yield gzip.GzipFile(mode='rb', fileobj=fileobj_after_rewind)  # type: ignore
        elif compression == 'LZO':
            # This might be useful to implement this:
            raise NotImplementedError('Records mover does not currently know how '
                                      'to decompress LZO files for inspection')
        elif compression == 'BZIP':
            yield bz2.BZ2File(mode='rb', filename=fileobj_after_rewind)
        else:
            _assert_never(compression)

def infer_newline_format(fileobj: IO[bytes],
                         encoding_hint: HintEncoding,
                         compression: HintCompression) ->\
        Optional[HintRecordTerminator]:
    with rewound_decompressed_fileobj(fileobj, compression) as fileobj:
        python_encoding = python_encoding_from_hint[encoding_hint]
        text_fileobj = io.TextIOWrapper(fileobj, encoding=python_encoding)
        try:
            if text_fileobj.newlines is None:  # ...and it almost certainly will be...
                text_fileobj.readline()  # read enough to know newline format
            if text_fileobj.newlines is not None:
                logger.info(f"Inferred record terminator as {repr(text_fileobj.newlines)}")
                return str(text_fileobj.newlines)
            else:
                logger.warning("Python could not determine newline format of file.")
                return None
        finally:
            text_fileobj.detach()

def sniff_encoding_hint(fileobj: IO[bytes]) -> Optional[HintEncoding]:
    with rewound_fileobj(fileobj) as fileobj:
        detector = chardet.UniversalDetector()
        while True:
            chunk = fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            detector.feed(chunk)
            if detector.done or len(chunk) < HINT_INFERENCE_SAMPLING_SIZE_BYTES:
                break
        detector.close()
        assert detector.result is not None
        if 'encoding' in detector.result:
            chardet_encoding = detector.result['encoding']
            if chardet_encoding in hint_encoding_from_chardet:
                return hint_encoding_from_chardet[chardet_encoding]
            else:
                logger.warning("Got unrecognized encoding from chardet "
                               f"sniffing: {detector.result}")
                return None
        else:
            logger.warning(f"Unable to sniff file encoding using chardet: {detector.result}")
            return None

def csv_hints_from_python(fileobj: IO[bytes],
                          record_terminator_hint: Optional[HintRecordTerminator],
                          encoding_hint: HintEncoding,
                          compression: HintCompression) -> PartialRecordsHints:
    with rewound_decompressed_fileobj(fileobj,
                                      compression) as fileobj:
        # Sniffer tries to determine quotechar, doublequote,
        python_encoding = python_encoding_from_hint[encoding_hint]
        # TextIOWrapper can only handle standard newline types:
        if record_terminator_hint not in [None, '\n', '\r', '\r\n']:
            logger.info("Unable to infer file with non-standard newlines "
                        f"using Python csv.Sniffer {repr(record_terminator_hint)}.")
            return {}
        text_fileobj = io.TextIOWrapper(fileobj,
                                        encoding=python_encoding,
                                        newline=record_terminator_hint)
        try:
            sniffer = csv.Sniffer()
            sample = text_fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            # the CSV sniffer's has_header() method seems to only
            # cope with DOS and UNIX newlines, not Mac.  So let's give it
            # UNIX newlines if we know enough to translate, since
            # we're not using it to sniff newline format anyway.
            if record_terminator_hint is not None and record_terminator_hint != '\n':
                sample_with_unix_newlines = sample.replace(record_terminator_hint, '\n')
            else:
                sample_with_unix_newlines = sample
            dialect = sniffer.sniff(sample_with_unix_newlines)
            header_row = sniffer.has_header(sample_with_unix_newlines)
            out: PartialRecordsHints = {
                'doublequote': True if dialect.doublequote else False,
                'field-delimiter': dialect.delimiter,
                'header-row': True if header_row else False,
            }
            if dialect.quotechar is not None:
                out['quotechar'] = dialect.quotechar
            logger.info(f"Python csv.Dialect sniffed: {out}")
            return out
        except csv.Error as e:
            if str(e) == 'Could not determine delimiter':
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                return {}
            else:
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                raise
        finally:
            text_fileobj.detach()

def csv_hints_from_pandas(fileobj: IO[bytes],
                          streaming_hints: PartialRecordsHints) -> PartialRecordsHints:
    import pandas

    def attempt_parse(quoting: HintQuoting) -> PartialRecordsHints:
        with rewound_fileobj(fileobj) as fresh_fileobj:
            current_hints = streaming_hints.copy()
            current_hints['quoting'] = quoting
            logger.info(f"Attempting to parse with quoting: {quoting}")
            with stream_csv(fresh_fileobj, current_hints):
                return {
                    'quoting': quoting
                }

    if 'quoting' in streaming_hints:
        return attempt_parse(streaming_hints['quoting'])
    else:
        # Pandas seems to parse quoting=minimal files just fine when
        # you pass in quoting=all, making this technique useless to
        # distinguish between minimal/nonnumeric/all, so we'll only
        try:
            return attempt_parse(quoting='minimal')
        except (pandas.errors.ParserError, pandas.errors.EmptyDataError):
            return attempt_parse(quoting=None)

def sniff_compression_hint(fileobj: IO[bytes]) -> HintCompression:
    print(f'Sniffing compression')
    with rewound_fileobj(fileobj) as fileobj_rewound:
        magic_dict: Dict[bytes, HintCompression] = {
            b"\x1f\x8b\x08": "GZIP",
            b"\x42\x5a\x68": "BZIP",
            # "\x50\x4b\x03\x04": "zip"
        }

        max_len = max(len(x) for x in magic_dict)

        file_start = fileobj_rewound.read(max_len)
        for magic, filetype in magic_dict.items():
            if file_start.startswith(magic):
                return filetype
        return None

def sniff_hints_from_fileobjs(fileobjs: List[IO[bytes]],
                              initial_hints: PartialRecordsHints) -> PartialRecordsHints:

def sniff_hints(fileobj: IO[bytes],
                initial_hints: PartialRecordsHints) -> PartialRecordsHints:
    # Major limitations:
    #  * If fileobj isn't rewindable, we can't sniff or we'd keep you
    #    from being able to use later.
    #  * We can't sniff from LZO files yet.
    #  * No detection of 'escape' or date/time format hints.
    #  * Only limited detection of 'quoting' hint.
    try:
        # We'll need to determine compression and encoding to be able
        # to convert from bytes to characters and figure out the rest:
        if 'compression' not in initial_hints:
            compression_hint = sniff_compression_hint(fileobj)
        else:
            compression_hint = initial_hints['compression']
        if 'encoding' not in initial_hints:
            encoding_hint = sniff_encoding_hint(fileobj)
        else:
            encoding_hint = initial_hints['encoding']
        # If guessing was inconclusive, default to UTF8
        final_encoding_hint: HintEncoding = (encoding_hint or 'UTF8')

        # Now we can figure out what type of newlines are used in the
        record_terminator_hint: Optional[HintRecordTerminator] = None
        if 'record-terminator' in initial_hints:
            record_terminator_hint = initial_hints['record-terminator']
        else:
            record_terminator_hint = infer_newline_format(fileobj,
                                                          final_encoding_hint,
                                                          compression_hint)

        # Now we have enough to study each line of the file, Python's
        # csv.Sniffer() can teach us some things about how each field
        # is encoded and whether there's a header:
        other_inferred_csv_hints = {}
        if record_terminator_hint is not None:
            other_inferred_csv_hints['record-terminator'] = record_terminator_hint
            python_inferred_hints = csv_hints_from_python(fileobj,
                                                          record_terminator_hint,
                                                          final_encoding_hint,
                                                          compression_hint)
        else:
            python_inferred_hints = {}

        # Pandas can both validate that we chose correctly by parsing
        # the file using what we have so far, and give us a crude shot
        # at finding a working 'quoting' hint:
        streaming_hints = initial_hints.copy()
        streaming_hints['compression'] = compression_hint
        if encoding_hint is not None:
            streaming_hints['encoding'] = encoding_hint
        streaming_hints.update(python_inferred_hints)
        pandas_inferred_hints = csv_hints_from_pandas(fileobj, streaming_hints)

        # Let's combine these together and present back a refined
        # version of the initial hints:
        out: PartialRecordsHints = {
            'compression': compression_hint,
            **pandas_inferred_hints,  # type: ignore
            **python_inferred_hints,
            'encoding': final_encoding_hint,
            **other_inferred_csv_hints,
            **initial_hints
        }
        logger.info(f"Inferred hints from combined sources: {out}")
        return out
    except OSError:
        logger.warning("Could not sniff hints, as stream is not rewindable")
        return {}
    except NotImplementedError as e:
        logger.warning(f"Could not sniff hints due to current limitations in records mover: {e}")
        return {}


while True:
            chunk = fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            detector.feed(chunk)
            if detector.done or len(chunk) < HINT_INFERENCE_SAMPLING_SIZE_BYTES:
                break

import chardet
from contextlib import contextmanager
from . import PartialRecordsHints
from .csv_streamer import stream_csv, python_encoding_from_hint
import io
import csv
import gzip
import bz2
from .types import HintEncoding, HintRecordTerminator, HintQuoting, HintCompression
from .conversions import hint_encoding_from_chardet
from typing import List, IO, Optional, Iterator, Dict
from records_mover.utils.rewound_fileobj import rewound_fileobj
from records_mover.mover_types import _assert_never
import logging

logger = logging.getLogger(__name__)

HINT_INFERENCE_SAMPLING_SIZE_BYTES = 1024

@contextmanager
def rewound_decompressed_fileobj(fileobj: IO[bytes],
                                 compression: HintCompression) -> Iterator[IO[bytes]]:
    with rewound_fileobj(fileobj) as fileobj_after_rewind:
        if compression is None:
            yield fileobj
        elif compression == 'GZIP':
            yield gzip.GzipFile(mode='rb', fileobj=fileobj_after_rewind)  # type: ignore
        elif compression == 'LZO':
            # This might be useful to implement this:
            raise NotImplementedError('Records mover does not currently know how '
                                      'to decompress LZO files for inspection')
        elif compression == 'BZIP':
            yield bz2.BZ2File(mode='rb', filename=fileobj_after_rewind)
        else:
            _assert_never(compression)

def infer_newline_format(fileobj: IO[bytes],
                         encoding_hint: HintEncoding,
                         compression: HintCompression) ->\
        Optional[HintRecordTerminator]:
    with rewound_decompressed_fileobj(fileobj, compression) as fileobj:
        python_encoding = python_encoding_from_hint[encoding_hint]
        text_fileobj = io.TextIOWrapper(fileobj, encoding=python_encoding)
        try:
            if text_fileobj.newlines is None:  # ...and it almost certainly will be...
                text_fileobj.readline()  # read enough to know newline format
            if text_fileobj.newlines is not None:
                logger.info(f"Inferred record terminator as {repr(text_fileobj.newlines)}")
                return str(text_fileobj.newlines)
            else:
                logger.warning("Python could not determine newline format of file.")
                return None
        finally:
            text_fileobj.detach()

def sniff_encoding_hint(fileobj: IO[bytes]) -> Optional[HintEncoding]:
    with rewound_fileobj(fileobj) as fileobj:
        detector = chardet.UniversalDetector()

        detector.close()
        assert detector.result is not None
        if 'encoding' in detector.result:
            chardet_encoding = detector.result['encoding']
            if chardet_encoding in hint_encoding_from_chardet:
                return hint_encoding_from_chardet[chardet_encoding]
            else:
                logger.warning("Got unrecognized encoding from chardet "
                               f"sniffing: {detector.result}")
                return None
        else:
            logger.warning(f"Unable to sniff file encoding using chardet: {detector.result}")
            return None

def csv_hints_from_python(fileobj: IO[bytes],
                          record_terminator_hint: Optional[HintRecordTerminator],
                          encoding_hint: HintEncoding,
                          compression: HintCompression) -> PartialRecordsHints:
    with rewound_decompressed_fileobj(fileobj,
                                      compression) as fileobj:
        # Sniffer tries to determine quotechar, doublequote,
        python_encoding = python_encoding_from_hint[encoding_hint]
        # TextIOWrapper can only handle standard newline types:
        if record_terminator_hint not in [None, '\n', '\r', '\r\n']:
            logger.info("Unable to infer file with non-standard newlines "
                        f"using Python csv.Sniffer {repr(record_terminator_hint)}.")
            return {}
        text_fileobj = io.TextIOWrapper(fileobj,
                                        encoding=python_encoding,
                                        newline=record_terminator_hint)
        try:
            sniffer = csv.Sniffer()
            sample = text_fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            # the CSV sniffer's has_header() method seems to only
            # cope with DOS and UNIX newlines, not Mac.  So let's give it
            # UNIX newlines if we know enough to translate, since
            # we're not using it to sniff newline format anyway.
            if record_terminator_hint is not None and record_terminator_hint != '\n':
                sample_with_unix_newlines = sample.replace(record_terminator_hint, '\n')
            else:
                sample_with_unix_newlines = sample
            dialect = sniffer.sniff(sample_with_unix_newlines)
            header_row = sniffer.has_header(sample_with_unix_newlines)
            out: PartialRecordsHints = {
                'doublequote': True if dialect.doublequote else False,
                'field-delimiter': dialect.delimiter,
                'header-row': True if header_row else False,
            }
            if dialect.quotechar is not None:
                out['quotechar'] = dialect.quotechar
            logger.info(f"Python csv.Dialect sniffed: {out}")
            return out
        except csv.Error as e:
            if str(e) == 'Could not determine delimiter':
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                return {}
            else:
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                raise
        finally:
            text_fileobj.detach()

def csv_hints_from_pandas(fileobj: IO[bytes],
                          streaming_hints: PartialRecordsHints) -> PartialRecordsHints:
    import pandas

    def attempt_parse(quoting: HintQuoting) -> PartialRecordsHints:
        with rewound_fileobj(fileobj) as fresh_fileobj:
            current_hints = streaming_hints.copy()
            current_hints['quoting'] = quoting
            logger.info(f"Attempting to parse with quoting: {quoting}")
            with stream_csv(fresh_fileobj, current_hints):
                return {
                    'quoting': quoting
                }

    if 'quoting' in streaming_hints:
        return attempt_parse(streaming_hints['quoting'])
    else:
        # Pandas seems to parse quoting=minimal files just fine when
        # you pass in quoting=all, making this technique useless to
        # distinguish between minimal/nonnumeric/all, so we'll only
        try:
            return attempt_parse(quoting='minimal')
        except (pandas.errors.ParserError, pandas.errors.EmptyDataError):
            return attempt_parse(quoting=None)

def sniff_compression_hint(fileobj: IO[bytes]) -> HintCompression:
    print(f'Sniffing compression')
    with rewound_fileobj(fileobj) as fileobj_rewound:
        magic_dict: Dict[bytes, HintCompression] = {
            b"\x1f\x8b\x08": "GZIP",
            b"\x42\x5a\x68": "BZIP",
            # "\x50\x4b\x03\x04": "zip"
        }

        max_len = max(len(x) for x in magic_dict)

        file_start = fileobj_rewound.read(max_len)
        for magic, filetype in magic_dict.items():
            if file_start.startswith(magic):
                return filetype
        return None

def sniff_hints_from_fileobjs(fileobjs: List[IO[bytes]],
                              initial_hints: PartialRecordsHints) -> PartialRecordsHints:
    if len(fileobjs) != 1:
        raise NotImplementedError('Cannot currently sniff hints from multiple '
                                  'files--please provide hints')
    fileobj = fileobjs[0]
    hints = sniff_hints(fileobj, initial_hints=initial_hints)
    return hints

def sniff_hints(fileobj: IO[bytes],
                initial_hints: PartialRecordsHints) -> PartialRecordsHints:
    # Major limitations:
    #  * If fileobj isn't rewindable, we can't sniff or we'd keep you
    #    from being able to use later.
    #  * We can't sniff from LZO files yet.
    #  * No detection of 'escape' or date/time format hints.
    #  * Only limited detection of 'quoting' hint.
    try:
        # We'll need to determine compression and encoding to be able
        # to convert from bytes to characters and figure out the rest:
        if 'compression' not in initial_hints:
            compression_hint = sniff_compression_hint(fileobj)
        else:
            compression_hint = initial_hints['compression']
        if 'encoding' not in initial_hints:
            encoding_hint = sniff_encoding_hint(fileobj)
        else:
            encoding_hint = initial_hints['encoding']
        # If guessing was inconclusive, default to UTF8
        final_encoding_hint: HintEncoding = (encoding_hint or 'UTF8')

        # Now we can figure out what type of newlines are used in the
        record_terminator_hint: Optional[HintRecordTerminator] = None
        if 'record-terminator' in initial_hints:
            record_terminator_hint = initial_hints['record-terminator']
        else:
            record_terminator_hint = infer_newline_format(fileobj,
                                                          final_encoding_hint,
                                                          compression_hint)

        # Now we have enough to study each line of the file, Python's
        # csv.Sniffer() can teach us some things about how each field
        # is encoded and whether there's a header:
        other_inferred_csv_hints = {}
        if record_terminator_hint is not None:
            other_inferred_csv_hints['record-terminator'] = record_terminator_hint
            python_inferred_hints = csv_hints_from_python(fileobj,
                                                          record_terminator_hint,
                                                          final_encoding_hint,
                                                          compression_hint)
        else:
            python_inferred_hints = {}

        # Pandas can both validate that we chose correctly by parsing
        # the file using what we have so far, and give us a crude shot
        # at finding a working 'quoting' hint:
        streaming_hints = initial_hints.copy()
        streaming_hints['compression'] = compression_hint
        if encoding_hint is not None:
            streaming_hints['encoding'] = encoding_hint
        streaming_hints.update(python_inferred_hints)
        pandas_inferred_hints = csv_hints_from_pandas(fileobj, streaming_hints)

        # Let's combine these together and present back a refined
        # version of the initial hints:
        out: PartialRecordsHints = {
            'compression': compression_hint,
            **pandas_inferred_hints,  # type: ignore
            **python_inferred_hints,
            'encoding': final_encoding_hint,
            **other_inferred_csv_hints,
            **initial_hints
        }
        logger.info(f"Inferred hints from combined sources: {out}")
        return out
    except OSError:
        logger.warning("Could not sniff hints, as stream is not rewindable")
        return {}
    except NotImplementedError as e:
        logger.warning(f"Could not sniff hints due to current limitations in records mover: {e}")
        return {}


try:
            return attempt_parse(quoting='minimal')
        except (pandas.errors.ParserError, pandas.errors.EmptyDataError):
            return attempt_parse(quoting=None)

import chardet
from contextlib import contextmanager
from . import PartialRecordsHints
from .csv_streamer import stream_csv, python_encoding_from_hint
import io
import csv
import gzip
import bz2
from .types import HintEncoding, HintRecordTerminator, HintQuoting, HintCompression
from .conversions import hint_encoding_from_chardet
from typing import List, IO, Optional, Iterator, Dict
from records_mover.utils.rewound_fileobj import rewound_fileobj
from records_mover.mover_types import _assert_never
import logging

logger = logging.getLogger(__name__)

HINT_INFERENCE_SAMPLING_SIZE_BYTES = 1024

@contextmanager
def rewound_decompressed_fileobj(fileobj: IO[bytes],
                                 compression: HintCompression) -> Iterator[IO[bytes]]:
    with rewound_fileobj(fileobj) as fileobj_after_rewind:
        if compression is None:
            yield fileobj
        elif compression == 'GZIP':
            yield gzip.GzipFile(mode='rb', fileobj=fileobj_after_rewind)  # type: ignore
        elif compression == 'LZO':
            # This might be useful to implement this:
            raise NotImplementedError('Records mover does not currently know how '
                                      'to decompress LZO files for inspection')
        elif compression == 'BZIP':
            yield bz2.BZ2File(mode='rb', filename=fileobj_after_rewind)
        else:
            _assert_never(compression)

def infer_newline_format(fileobj: IO[bytes],
                         encoding_hint: HintEncoding,
                         compression: HintCompression) ->\
        Optional[HintRecordTerminator]:
    with rewound_decompressed_fileobj(fileobj, compression) as fileobj:
        python_encoding = python_encoding_from_hint[encoding_hint]
        text_fileobj = io.TextIOWrapper(fileobj, encoding=python_encoding)
        try:
            if text_fileobj.newlines is None:  # ...and it almost certainly will be...
                text_fileobj.readline()  # read enough to know newline format
            if text_fileobj.newlines is not None:
                logger.info(f"Inferred record terminator as {repr(text_fileobj.newlines)}")
                return str(text_fileobj.newlines)
            else:
                logger.warning("Python could not determine newline format of file.")
                return None
        finally:
            text_fileobj.detach()

def sniff_encoding_hint(fileobj: IO[bytes]) -> Optional[HintEncoding]:
    with rewound_fileobj(fileobj) as fileobj:
        detector = chardet.UniversalDetector()
        while True:
            chunk = fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            detector.feed(chunk)
            if detector.done or len(chunk) < HINT_INFERENCE_SAMPLING_SIZE_BYTES:
                break
        detector.close()
        assert detector.result is not None
        if 'encoding' in detector.result:
            chardet_encoding = detector.result['encoding']
            if chardet_encoding in hint_encoding_from_chardet:
                return hint_encoding_from_chardet[chardet_encoding]
            else:
                logger.warning("Got unrecognized encoding from chardet "
                               f"sniffing: {detector.result}")
                return None
        else:
            logger.warning(f"Unable to sniff file encoding using chardet: {detector.result}")
            return None

def csv_hints_from_python(fileobj: IO[bytes],
                          record_terminator_hint: Optional[HintRecordTerminator],
                          encoding_hint: HintEncoding,
                          compression: HintCompression) -> PartialRecordsHints:
    with rewound_decompressed_fileobj(fileobj,
                                      compression) as fileobj:
        # Sniffer tries to determine quotechar, doublequote,
        python_encoding = python_encoding_from_hint[encoding_hint]
        # TextIOWrapper can only handle standard newline types:
        if record_terminator_hint not in [None, '\n', '\r', '\r\n']:
            logger.info("Unable to infer file with non-standard newlines "
                        f"using Python csv.Sniffer {repr(record_terminator_hint)}.")
            return {}
        text_fileobj = io.TextIOWrapper(fileobj,
                                        encoding=python_encoding,
                                        newline=record_terminator_hint)
        try:
            sniffer = csv.Sniffer()
            sample = text_fileobj.read(HINT_INFERENCE_SAMPLING_SIZE_BYTES)
            # the CSV sniffer's has_header() method seems to only
            # cope with DOS and UNIX newlines, not Mac.  So let's give it
            # UNIX newlines if we know enough to translate, since
            # we're not using it to sniff newline format anyway.
            if record_terminator_hint is not None and record_terminator_hint != '\n':
                sample_with_unix_newlines = sample.replace(record_terminator_hint, '\n')
            else:
                sample_with_unix_newlines = sample
            dialect = sniffer.sniff(sample_with_unix_newlines)
            header_row = sniffer.has_header(sample_with_unix_newlines)
            out: PartialRecordsHints = {
                'doublequote': True if dialect.doublequote else False,
                'field-delimiter': dialect.delimiter,
                'header-row': True if header_row else False,
            }
            if dialect.quotechar is not None:
                out['quotechar'] = dialect.quotechar
            logger.info(f"Python csv.Dialect sniffed: {out}")
            return out
        except csv.Error as e:
            if str(e) == 'Could not determine delimiter':
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                return {}
            else:
                logger.info(f"Error from csv.Sniffer--potential single-field file: {str(e)}")
                raise
        finally:
            text_fileobj.detach()

def csv_hints_from_pandas(fileobj: IO[bytes],
                          streaming_hints: PartialRecordsHints) -> PartialRecordsHints:
    import pandas

    def attempt_parse(quoting: HintQuoting) -> PartialRecordsHints:
        with rewound_fileobj(fileobj) as fresh_fileobj:
            current_hints = streaming_hints.copy()
            current_hints['quoting'] = quoting
            logger.info(f"Attempting to parse with quoting: {quoting}")
            with stream_csv(fresh_fileobj, current_hints):
                return {
                    'quoting': quoting
                }

    if 'quoting' in streaming_hints:
        return attempt_parse(streaming_hints['quoting'])
    else:
        # Pandas seems to parse quoting=minimal files just fine when
        # you pass in quoting=all, making this technique useless to
        # distinguish between minimal/nonnumeric/all, so we'll only

def sniff_compression_hint(fileobj: IO[bytes]) -> HintCompression:
    print(f'Sniffing compression')
    with rewound_fileobj(fileobj) as fileobj_rewound:
        magic_dict: Dict[bytes, HintCompression] = {
            b"\x1f\x8b\x08": "GZIP",
            b"\x42\x5a\x68": "BZIP",
            # "\x50\x4b\x03\x04": "zip"
        }

        max_len = max(len(x) for x in magic_dict)

        file_start = fileobj_rewound.read(max_len)
        for magic, filetype in magic_dict.items():
            if file_start.startswith(magic):
                return filetype
        return None

def sniff_hints_from_fileobjs(fileobjs: List[IO[bytes]],
                              initial_hints: PartialRecordsHints) -> PartialRecordsHints:
    if len(fileobjs) != 1:
        raise NotImplementedError('Cannot currently sniff hints from multiple '
                                  'files--please provide hints')
    fileobj = fileobjs[0]
    hints = sniff_hints(fileobj, initial_hints=initial_hints)
    return hints

def sniff_hints(fileobj: IO[bytes],
                initial_hints: PartialRecordsHints) -> PartialRecordsHints:
    # Major limitations:
    #  * If fileobj isn't rewindable, we can't sniff or we'd keep you
    #    from being able to use later.
    #  * We can't sniff from LZO files yet.
    #  * No detection of 'escape' or date/time format hints.
    #  * Only limited detection of 'quoting' hint.
    try:
        # We'll need to determine compression and encoding to be able
        # to convert from bytes to characters and figure out the rest:
        if 'compression' not in initial_hints:
            compression_hint = sniff_compression_hint(fileobj)
        else:
            compression_hint = initial_hints['compression']
        if 'encoding' not in initial_hints:
            encoding_hint = sniff_encoding_hint(fileobj)
        else:
            encoding_hint = initial_hints['encoding']
        # If guessing was inconclusive, default to UTF8
        final_encoding_hint: HintEncoding = (encoding_hint or 'UTF8')

        # Now we can figure out what type of newlines are used in the
        record_terminator_hint: Optional[HintRecordTerminator] = None
        if 'record-terminator' in initial_hints:
            record_terminator_hint = initial_hints['record-terminator']
        else:
            record_terminator_hint = infer_newline_format(fileobj,
                                                          final_encoding_hint,
                                                          compression_hint)

        # Now we have enough to study each line of the file, Python's
        # csv.Sniffer() can teach us some things about how each field
        # is encoded and whether there's a header:
        other_inferred_csv_hints = {}
        if record_terminator_hint is not None:
            other_inferred_csv_hints['record-terminator'] = record_terminator_hint
            python_inferred_hints = csv_hints_from_python(fileobj,
                                                          record_terminator_hint,
                                                          final_encoding_hint,
                                                          compression_hint)
        else:
            python_inferred_hints = {}

        # Pandas can both validate that we chose correctly by parsing
        # the file using what we have so far, and give us a crude shot
        # at finding a working 'quoting' hint:
        streaming_hints = initial_hints.copy()
        streaming_hints['compression'] = compression_hint
        if encoding_hint is not None:
            streaming_hints['encoding'] = encoding_hint
        streaming_hints.update(python_inferred_hints)
        pandas_inferred_hints = csv_hints_from_pandas(fileobj, streaming_hints)

        # Let's combine these together and present back a refined
        # version of the initial hints:
        out: PartialRecordsHints = {
            'compression': compression_hint,
            **pandas_inferred_hints,  # type: ignore
            **python_inferred_hints,
            'encoding': final_encoding_hint,
            **other_inferred_csv_hints,
            **initial_hints
        }
        logger.info(f"Inferred hints from combined sources: {out}")
        return out
    except OSError:
        logger.warning("Could not sniff hints, as stream is not rewindable")
        return {}
    except NotImplementedError as e:
        logger.warning(f"Could not sniff hints due to current limitations in records mover: {e}")
        return {}


z = self._z(x)
    return - z - math_ops.exp(-z)

"""The Gumbel distribution class."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_shape
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import check_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops.distributions import distribution

class _Gumbel(distribution.Distribution):
  """The scalar Gumbel distribution with location `loc` and `scale` parameters.

  The probability density function (pdf) of this distribution is,

  ```none
  pdf(x; mu, sigma) = exp(-(x - mu) / sigma - exp(-(x - mu) / sigma))
  ```

  where `loc = mu` and `scale = sigma`.

  The cumulative density function of this distribution is,

  ```cdf(x; mu, sigma) = exp(-exp(-(x - mu) / sigma))```

  The Gumbel distribution is a member of the [location-scale family](
  http://example.com), i.e., it can be
  constructed as,

  ```none
  X ~ Gumbel(loc=0, scale=1)
  Y = loc + scale * X
  ```

  Examples of initialization of one or a batch of distributions.

  ```python
  # Define a single scalar Gumbel distribution.
  dist = tf.contrib.distributions.Gumbel(loc=0., scale=3.)

  # Evaluate the cdf at 1, returning a scalar.
  dist.cdf(1.)

  # Define a batch of two scalar valued Gumbels.
  # The first has mean 1 and scale 11, the second 2 and 22.
  dist = tf.contrib.distributions.Gumbel(loc=[1, 2.], scale=[11, 22.])

  # Evaluate the pdf of the first distribution on 0, and the second on 1.5,
  # returning a length two tensor.
  dist.prob([0, 1.5])

  dist.sample([3])
  ```

  Arguments are broadcast when possible.

  ```python
  # Define a batch of two scalar valued Logistics.
  # Both have mean 1, but different scales.
  dist = tf.contrib.distributions.Gumbel(loc=1., scale=[11, 22.])

  # Evaluate the pdf of both distributions on the same point, 3.0,
  # returning a length 2 tensor.
  dist.prob(3.0)
  ```

  """

  def __init__(self,
               loc,
               scale,
               validate_args=False,
               allow_nan_stats=True,
               name="Gumbel"):
    """Construct Gumbel distributions with location and scale `loc` and `scale`.

    The parameters `loc` and `scale` must be shaped in a way that supports
    broadcasting (e.g. `loc + scale` is a valid operation).

    Args:
      loc: Floating point tensor, the means of the distribution(s).
      scale: Floating point tensor, the scales of the distribution(s).
        scale must contain only positive values.
      validate_args: Python `bool`, default `False`. When `True` distribution
        parameters are checked for validity despite possibly degrading runtime
        performance. When `False` invalid inputs may silently render incorrect
        outputs.
      allow_nan_stats: Python `bool`, default `True`. When `True`,
        statistics (e.g., mean, mode, variance) use the value "`NaN`" to
        indicate the result is undefined. When `False`, an exception is raised
        if one or more of the statistic's batch members are undefined.
      name: Python `str` name prefixed to Ops created by this class.

    Raises:
      TypeError: if loc and scale are different dtypes.
    """
    parameters = locals()
    with ops.name_scope(name, values=[loc, scale]):
      with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                    validate_args else []):
        self._loc = array_ops.identity(loc, name="loc")
        self._scale = array_ops.identity(scale, name="scale")
        check_ops.assert_same_float_dtype([self._loc, self._scale])
    super(_Gumbel, self).__init__(
        dtype=self._scale.dtype,
        reparameterization_type=distribution.FULLY_REPARAMETERIZED,
        validate_args=validate_args,
        allow_nan_stats=allow_nan_stats,
        parameters=parameters,
        graph_parents=[self._loc, self._scale],
        name=name)

  @staticmethod
  def _param_shapes(sample_shape):
    return dict(
        zip(("loc", "scale"), ([ops.convert_to_tensor(
            sample_shape, dtype=dtypes.int32)] * 2)))

  @property
  def loc(self):
    """Distribution parameter for the location."""
    return self._loc

  @property
  def scale(self):
    """Distribution parameter for scale."""
    return self._scale

  def _batch_shape_tensor(self):
    return array_ops.broadcast_dynamic_shape(
        array_ops.shape(self.loc), array_ops.shape(self.scale))

  def _batch_shape(self):
    return array_ops.broadcast_static_shape(
        self.loc.get_shape(), self.scale.get_shape())

  def _event_shape_tensor(self):
    return constant_op.constant([], dtype=dtypes.int32)

  def _event_shape(self):
    return tensor_shape.scalar()

  def _sample_n(self, n, seed=None):
    # Uniform variates must be sampled from the open-interval `(0, 1)` rather
    # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny`
    # because it is the smallest, positive, "normal" number. A "normal" number
    # is such that the mantissa has an implicit leading 1. Normal, positive
    # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In
    # this case, a subnormal number (i.e., np.nextafter) can cause us to sample
    uniform = random_ops.random_uniform(
        shape=array_ops.concat([[n], self.batch_shape_tensor()], 0),
        minval=np.finfo(self.dtype.as_numpy_dtype).tiny,
        maxval=1.,
        dtype=self.dtype,
        seed=seed)
    sampled = -math_ops.log(-math_ops.log(uniform))
    return sampled * self.scale + self.loc

  def _log_prob(self, x):
    return self._log_unnormalized_prob(x) - self._log_normalization()

  def _prob(self, x):
    return math_ops.exp(self._log_prob(x))

  def _log_cdf(self, x):
    return -math_ops.exp(-self._z(x))

  def _cdf(self, x):
    return math_ops.exp(-math_ops.exp(-self._z(x)))

  def _log_unnormalized_prob(self, x):

  def _log_normalization(self):
    return math_ops.log(self.scale)

  def _entropy(self):
    # Use broadcasting rules to calculate the full broadcast sigma.
    scale = self.scale * array_ops.ones_like(self.loc)
    return 1 + math_ops.log(scale) + np.euler_gamma

  def _mean(self):
    return self.loc + self.scale * np.euler_gamma

  def _stddev(self):
    return self.scale * array_ops.ones_like(self.loc) * math.pi / math.sqrt(6)

  def _mode(self):
    return self.loc * array_ops.ones_like(self.scale)

  def _z(self, x):
    """Standardize input `x` to a unit logistic."""
    with ops.name_scope("standardize", values=[x]):
      return (x - self.loc) / self.scale


"""The Gumbel distribution class."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
import numpy as np
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_shape
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import check_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops.distributions import distribution

class _Gumbel(distribution.Distribution):
  """The scalar Gumbel distribution with location `loc` and `scale` parameters.

  The probability density function (pdf) of this distribution is,

  ```none
  pdf(x; mu, sigma) = exp(-(x - mu) / sigma - exp(-(x - mu) / sigma))
  ```

  where `loc = mu` and `scale = sigma`.

  The cumulative density function of this distribution is,

  ```cdf(x; mu, sigma) = exp(-exp(-(x - mu) / sigma))```

  The Gumbel distribution is a member of the [location-scale family](
  http://example.com), i.e., it can be
  constructed as,

  ```none
  X ~ Gumbel(loc=0, scale=1)
  Y = loc + scale * X
  ```

  Examples of initialization of one or a batch of distributions.

  ```python
  # Define a single scalar Gumbel distribution.
  dist = tf.contrib.distributions.Gumbel(loc=0., scale=3.)

  # Evaluate the cdf at 1, returning a scalar.
  dist.cdf(1.)

  # Define a batch of two scalar valued Gumbels.
  # The first has mean 1 and scale 11, the second 2 and 22.
  dist = tf.contrib.distributions.Gumbel(loc=[1, 2.], scale=[11, 22.])

  # Evaluate the pdf of the first distribution on 0, and the second on 1.5,
  # returning a length two tensor.
  dist.prob([0, 1.5])

  dist.sample([3])
  ```

  Arguments are broadcast when possible.

  ```python
  # Define a batch of two scalar valued Logistics.
  # Both have mean 1, but different scales.
  dist = tf.contrib.distributions.Gumbel(loc=1., scale=[11, 22.])

  # Evaluate the pdf of both distributions on the same point, 3.0,
  # returning a length 2 tensor.
  dist.prob(3.0)
  ```

  """

  def __init__(self,
               loc,
               scale,
               validate_args=False,
               allow_nan_stats=True,
               name="Gumbel"):
    """Construct Gumbel distributions with location and scale `loc` and `scale`.

    The parameters `loc` and `scale` must be shaped in a way that supports
    broadcasting (e.g. `loc + scale` is a valid operation).

    Args:
      loc: Floating point tensor, the means of the distribution(s).
      scale: Floating point tensor, the scales of the distribution(s).
        scale must contain only positive values.
      validate_args: Python `bool`, default `False`. When `True` distribution
        parameters are checked for validity despite possibly degrading runtime
        performance. When `False` invalid inputs may silently render incorrect
        outputs.
      allow_nan_stats: Python `bool`, default `True`. When `True`,
        statistics (e.g., mean, mode, variance) use the value "`NaN`" to
        indicate the result is undefined. When `False`, an exception is raised
        if one or more of the statistic's batch members are undefined.
      name: Python `str` name prefixed to Ops created by this class.

    Raises:
      TypeError: if loc and scale are different dtypes.
    """
    parameters = locals()
    with ops.name_scope(name, values=[loc, scale]):
      with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                    validate_args else []):
        self._loc = array_ops.identity(loc, name="loc")
        self._scale = array_ops.identity(scale, name="scale")
        check_ops.assert_same_float_dtype([self._loc, self._scale])
    super(_Gumbel, self).__init__(
        dtype=self._scale.dtype,
        reparameterization_type=distribution.FULLY_REPARAMETERIZED,
        validate_args=validate_args,
        allow_nan_stats=allow_nan_stats,
        parameters=parameters,
        graph_parents=[self._loc, self._scale],
        name=name)

  @staticmethod
  def _param_shapes(sample_shape):
    return dict(
        zip(("loc", "scale"), ([ops.convert_to_tensor(
            sample_shape, dtype=dtypes.int32)] * 2)))

  @property
  def loc(self):
    """Distribution parameter for the location."""
    return self._loc

  @property
  def scale(self):
    """Distribution parameter for scale."""

  def _batch_shape_tensor(self):
    return array_ops.broadcast_dynamic_shape(
        array_ops.shape(self.loc), array_ops.shape(self.scale))

  def _batch_shape(self):
    return array_ops.broadcast_static_shape(
        self.loc.get_shape(), self.scale.get_shape())

  def _event_shape_tensor(self):
    return constant_op.constant([], dtype=dtypes.int32)

  def _event_shape(self):
    return tensor_shape.scalar()

  def _sample_n(self, n, seed=None):
    # Uniform variates must be sampled from the open-interval `(0, 1)` rather
    # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny`
    # because it is the smallest, positive, "normal" number. A "normal" number
    # is such that the mantissa has an implicit leading 1. Normal, positive
    # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In
    # this case, a subnormal number (i.e., np.nextafter) can cause us to sample
    uniform = random_ops.random_uniform(
        shape=array_ops.concat([[n], self.batch_shape_tensor()], 0),
        minval=np.finfo(self.dtype.as_numpy_dtype).tiny,
        maxval=1.,
        dtype=self.dtype,
        seed=seed)
    sampled = -math_ops.log(-math_ops.log(uniform))
    return sampled * self.scale + self.loc

  def _log_prob(self, x):
    return self._log_unnormalized_prob(x) - self._log_normalization()

  def _prob(self, x):
    return math_ops.exp(self._log_prob(x))

  def _log_cdf(self, x):
    return -math_ops.exp(-self._z(x))

  def _cdf(self, x):
    return math_ops.exp(-math_ops.exp(-self._z(x)))

  def _log_unnormalized_prob(self, x):
    z = self._z(x)
    return - z - math_ops.exp(-z)

  def _log_normalization(self):
    return math_ops.log(self.scale)

  def _entropy(self):
    # Use broadcasting rules to calculate the full broadcast sigma.
    scale = self.scale * array_ops.ones_like(self.loc)
    return 1 + math_ops.log(scale) + np.euler_gamma

  def _mean(self):
    return self.loc + self.scale * np.euler_gamma

  def _stddev(self):
    return self.scale * array_ops.ones_like(self.loc) * math.pi / math.sqrt(6)

  def _mode(self):
    return self.loc * array_ops.ones_like(self.scale)

  def _z(self, x):
    """Standardize input `x` to a unit logistic."""
    with ops.name_scope("standardize", values=[x]):
      return (x - self.loc) / self.scale


self.initmixin()
        self.name = 'Demo'

import mixin

class Demo(mixin.Mixin):
    __mixinname__ = 'Demo'
    def __init__(self):

def add(a, b):
    return a + b

mixin.setMixin('Demo', '1', 'aaaaaaaa')
mixin.setMixin('Demo', '2', (1,2,4))
mixin.setMixin('Demo', '2', (3,4))
mixin.setMixin('Demo', '3', {1:2})
mixin.setMixin('Demo', '3', {1:3})
mixin.setMixin('Demo', '5', 'asssssssssssssssssssssssssss')
mixin.setMixin('Demo', '5', (1,2,3,45))
mixin.setMixin('Demo', '5', 'assss')
mixin.setMixin('Demo', '6', [1,2,3])
mixin.setMixin('Demo', '5', [1,2])
mixin.setMixin('Demo', 'add', add)
mixin.setMixin('Demo', 'change', 'setup.change_package_fromLib')

def main():
    d = Demo()

    print mixin.__mixinset__

    print d
    print Demo

    print dir(d)
    print dir(Demo)

    print d.__dict__
    print Demo.__dict__

    print hasattr(d, 'add')
    print d.add(3, 4)

    print d.__mixins__

if __name__ == '__main__':
    main()


from __future__ import absolute_import, division, print_function
from libtbx import object_oriented_patterns as oop
from libtbx.test_utils import approx_equal, Exception_expected
from six.moves import range

def exercise_injector():
  class a(object):
    """ doc for a """
    def __init__(self, i): self.i = i
    def get(self): return self.i
    def set(self, i): self.i = i

  class a_extension(oop.injector, a):
    """ doc for extension of a """
    var = 1
    def get_square(self):

  assert a.var == 1
  o = a(2)
  assert o.get() == 2
  assert o.get_square() == 4

  class b(object):
    def __init__(self, i): self.i = i
    def get(self): return self.i-1

  class c(b):
    def get(self): return self.i + 1

  class c_extension(oop.injector, c):
    def get_square(self): return self.get() * self.get()

  o = c(-3)
  assert o.get() == -2
  assert o.get_square() == 4

  try:
    class d(a): pass
    class d_extension(oop.injector, d):
      def get_square(self): return 0
  except AssertionError as err:
    assert str(err) == "class d already has attribute 'get_square'"

def exercise_memoize():
  diagnostic = []
  def f(x):
    """ Documentation for function f """
    diagnostic.append('+')
    return x+1

  mf = oop.memoize(f)
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(1) == 2
  assert diagnostic == ['+']*2
  assert mf(0) == 1
  assert diagnostic == ['+']*2
  assert mf(1) == 2
  assert diagnostic == ['+']*2
  try:
    mf(x=1)
  except TypeError as e:
    pass
  else:
    raise Exception_expected
  assert mf.__doc__ == """ Documentation for function f """

def exercise_memoize_with_injector():
  class foo(object):
    def __init__(self, a):
      self.a = a
    def f(self, x):
      """ Documentation for method f """
      diagnostic.append('+')
      return self.a + x
    f = oop.memoize_method(f)

  diagnostic = []
  o1 = foo(1)
  o2 = foo(2)
  assert o1.f(4) == 5
  assert diagnostic == ['+']
  assert o2.f(5) == 7
  assert diagnostic == ['+']*2
  assert o1.f(2) == 3
  assert diagnostic == ['+']*3
  assert o1.f(4) == 5
  assert diagnostic == ['+']*3
  assert o2.f(5) == 7
  assert diagnostic == ['+']*3
  o3 = o1
  assert o3.f(4) == 5
  assert diagnostic == ['+']*3
  assert o3.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f.__doc__ == " Documentation for method f "

  class _foo_extension(oop.injector, foo):
    def g(self, n):
      return 'x'*n
    g = oop.memoize_method(g)

  efoo = foo(0)
  assert efoo.g(2) == "xx"
  assert efoo.g(3) == "xxx"
  assert efoo.g(2) == "xx"
  assert efoo._memoized_g.cached == {(2,):'xx', (3,): 'xxx'}

def exercise_null():
  n = oop.null("a", 1, [])
  assert not n
  assert isinstance(n.f, oop.null)
  assert isinstance(n.g(1, b=2), oop.null)
  h = n.h = 2
  assert h == 2
  assert isinstance(n.h, oop.null)
  assert isinstance(n[23085], oop.null)
  x = n[234] = 2
  assert x == 2
  assert isinstance(n[234], oop.null)

def exercise_journal():

  class test(object):

    def __init__(self):
      for i in range(10):
        self.x = i
      for k in range(5):
        self.z = k

  # create a subclass of test that journals x and y attributes
  class test1(oop.journal_mixin, test):
    __journal__ = ["x", "y"]

  class test2(oop.journal_mixin, test):
    __journal__ = ["x", "z"]
    __journal_suffix__ = "_journal"

  a = test1()
  assert a.x == 9
  assert approx_equal(a.x_history, list(range(10)))
  try: a.y
  except AttributeError: pass
  else: raise Exception_expected
  try: a.y_history
  except AttributeError: pass
  else: raise Exception_expected
  assert a.z == 4
  try: a.z_history
  except AttributeError: pass
  else: raise Exception_expected
  b = test2()
  assert approx_equal(b.x_journal, list(range(10)))
  assert approx_equal(b.z_journal, list(range(5)))
  del b.x
  try: b.x
  except AttributeError: pass
  else: raise Exception_expected
  try: b.x_journal
  except AttributeError: pass
  else: raise Exception_expected

def run():
  exercise_journal()
  exercise_null()
  exercise_memoize()
  exercise_injector()
  exercise_memoize_with_injector()

if __name__ == '__main__':
  run()


    diagnostic.append('+')
    return x+1

from __future__ import absolute_import, division, print_function
from libtbx import object_oriented_patterns as oop
from libtbx.test_utils import approx_equal, Exception_expected
from six.moves import range

def exercise_injector():
  class a(object):
    """ doc for a """
    def __init__(self, i): self.i = i
    def get(self): return self.i
    def set(self, i): self.i = i

  class a_extension(oop.injector, a):
    """ doc for extension of a """
    var = 1
    def get_square(self): return self.get() * self.get()

  assert a.var == 1
  o = a(2)
  assert o.get() == 2
  assert o.get_square() == 4

  class b(object):
    def __init__(self, i): self.i = i
    def get(self): return self.i-1

  class c(b):
    def get(self): return self.i + 1

  class c_extension(oop.injector, c):
    def get_square(self): return self.get() * self.get()

  o = c(-3)
  assert o.get() == -2
  assert o.get_square() == 4

  try:
    class d(a): pass
    class d_extension(oop.injector, d):
      def get_square(self): return 0
  except AssertionError as err:
    assert str(err) == "class d already has attribute 'get_square'"

def exercise_memoize():
  diagnostic = []
  def f(x):
    """ Documentation for function f """

  mf = oop.memoize(f)
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(1) == 2
  assert diagnostic == ['+']*2
  assert mf(0) == 1
  assert diagnostic == ['+']*2
  assert mf(1) == 2
  assert diagnostic == ['+']*2
  try:
    mf(x=1)
  except TypeError as e:
    pass
  else:
    raise Exception_expected
  assert mf.__doc__ == """ Documentation for function f """

def exercise_memoize_with_injector():
  class foo(object):
    def __init__(self, a):
      self.a = a
    def f(self, x):
      """ Documentation for method f """
      diagnostic.append('+')
      return self.a + x
    f = oop.memoize_method(f)

  diagnostic = []
  o1 = foo(1)
  o2 = foo(2)
  assert o1.f(4) == 5
  assert diagnostic == ['+']
  assert o2.f(5) == 7
  assert diagnostic == ['+']*2
  assert o1.f(2) == 3
  assert diagnostic == ['+']*3
  assert o1.f(4) == 5
  assert diagnostic == ['+']*3
  assert o2.f(5) == 7
  assert diagnostic == ['+']*3
  o3 = o1
  assert o3.f(4) == 5
  assert diagnostic == ['+']*3
  assert o3.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f.__doc__ == " Documentation for method f "

  class _foo_extension(oop.injector, foo):
    def g(self, n):
      return 'x'*n
    g = oop.memoize_method(g)

  efoo = foo(0)
  assert efoo.g(2) == "xx"
  assert efoo.g(3) == "xxx"
  assert efoo.g(2) == "xx"
  assert efoo._memoized_g.cached == {(2,):'xx', (3,): 'xxx'}

def exercise_null():
  n = oop.null("a", 1, [])
  assert not n
  assert isinstance(n.f, oop.null)
  assert isinstance(n.g(1, b=2), oop.null)
  h = n.h = 2
  assert h == 2
  assert isinstance(n.h, oop.null)
  assert isinstance(n[23085], oop.null)
  x = n[234] = 2
  assert x == 2
  assert isinstance(n[234], oop.null)

def exercise_journal():

  class test(object):

    def __init__(self):
      for i in range(10):
        self.x = i
      for k in range(5):
        self.z = k

  # create a subclass of test that journals x and y attributes
  class test1(oop.journal_mixin, test):
    __journal__ = ["x", "y"]

  class test2(oop.journal_mixin, test):
    __journal__ = ["x", "z"]
    __journal_suffix__ = "_journal"

  a = test1()
  assert a.x == 9
  assert approx_equal(a.x_history, list(range(10)))
  try: a.y
  except AttributeError: pass
  else: raise Exception_expected
  try: a.y_history
  except AttributeError: pass
  else: raise Exception_expected
  assert a.z == 4
  try: a.z_history
  except AttributeError: pass
  else: raise Exception_expected
  b = test2()
  assert approx_equal(b.x_journal, list(range(10)))
  assert approx_equal(b.z_journal, list(range(5)))
  del b.x
  try: b.x
  except AttributeError: pass
  else: raise Exception_expected
  try: b.x_journal
  except AttributeError: pass
  else: raise Exception_expected

def run():
  exercise_journal()
  exercise_null()
  exercise_memoize()
  exercise_injector()
  exercise_memoize_with_injector()

if __name__ == '__main__':
  run()


try:
    mf(x=1)
  except TypeError as e:
    pass
  else:
    raise Exception_expected

from __future__ import absolute_import, division, print_function
from libtbx import object_oriented_patterns as oop
from libtbx.test_utils import approx_equal, Exception_expected
from six.moves import range

def exercise_injector():
  class a(object):
    """ doc for a """
    def __init__(self, i): self.i = i
    def get(self): return self.i
    def set(self, i): self.i = i

  class a_extension(oop.injector, a):
    """ doc for extension of a """
    var = 1
    def get_square(self): return self.get() * self.get()

  assert a.var == 1
  o = a(2)
  assert o.get() == 2
  assert o.get_square() == 4

  class b(object):
    def __init__(self, i): self.i = i
    def get(self): return self.i-1

  class c(b):
    def get(self): return self.i + 1

  class c_extension(oop.injector, c):
    def get_square(self): return self.get() * self.get()

  o = c(-3)
  assert o.get() == -2
  assert o.get_square() == 4

  try:
    class d(a): pass
    class d_extension(oop.injector, d):
      def get_square(self): return 0
  except AssertionError as err:
    assert str(err) == "class d already has attribute 'get_square'"

def exercise_memoize():
  diagnostic = []
  def f(x):
    """ Documentation for function f """
    diagnostic.append('+')
    return x+1

  mf = oop.memoize(f)
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(0) == 1
  assert diagnostic == ['+']
  assert mf(1) == 2
  assert diagnostic == ['+']*2
  assert mf(0) == 1
  assert diagnostic == ['+']*2
  assert mf(1) == 2
  assert diagnostic == ['+']*2

  assert mf.__doc__ == """ Documentation for function f """

def exercise_memoize_with_injector():
  class foo(object):
    def __init__(self, a):
      self.a = a
    def f(self, x):
      """ Documentation for method f """
      diagnostic.append('+')
      return self.a + x
    f = oop.memoize_method(f)

  diagnostic = []
  o1 = foo(1)
  o2 = foo(2)
  assert o1.f(4) == 5
  assert diagnostic == ['+']
  assert o2.f(5) == 7
  assert diagnostic == ['+']*2
  assert o1.f(2) == 3
  assert diagnostic == ['+']*3
  assert o1.f(4) == 5
  assert diagnostic == ['+']*3
  assert o2.f(5) == 7
  assert diagnostic == ['+']*3
  o3 = o1
  assert o3.f(4) == 5
  assert diagnostic == ['+']*3
  assert o3.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f(1) == 2
  assert diagnostic == ['+']*4
  assert o1.f.__doc__ == " Documentation for method f "

  class _foo_extension(oop.injector, foo):
    def g(self, n):
      return 'x'*n
    g = oop.memoize_method(g)

  efoo = foo(0)
  assert efoo.g(2) == "xx"
  assert efoo.g(3) == "xxx"
  assert efoo.g(2) == "xx"
  assert efoo._memoized_g.cached == {(2,):'xx', (3,): 'xxx'}

def exercise_null():
  n = oop.null("a", 1, [])
  assert not n
  assert isinstance(n.f, oop.null)
  assert isinstance(n.g(1, b=2), oop.null)
  h = n.h = 2
  assert h == 2
  assert isinstance(n.h, oop.null)
  assert isinstance(n[23085], oop.null)
  x = n[234] = 2
  assert x == 2
  assert isinstance(n[234], oop.null)

def exercise_journal():

  class test(object):

    def __init__(self):
      for i in range(10):
        self.x = i
      for k in range(5):
        self.z = k

  # create a subclass of test that journals x and y attributes
  class test1(oop.journal_mixin, test):
    __journal__ = ["x", "y"]

  class test2(oop.journal_mixin, test):
    __journal__ = ["x", "z"]
    __journal_suffix__ = "_journal"

  a = test1()
  assert a.x == 9
  assert approx_equal(a.x_history, list(range(10)))
  try: a.y
  except AttributeError: pass
  else: raise Exception_expected
  try: a.y_history
  except AttributeError: pass
  else: raise Exception_expected
  assert a.z == 4
  try: a.z_history
  except AttributeError: pass
  else: raise Exception_expected
  b = test2()
  assert approx_equal(b.x_journal, list(range(10)))
  assert approx_equal(b.z_journal, list(range(5)))
  del b.x
  try: b.x
  except AttributeError: pass
  else: raise Exception_expected
  try: b.x_journal
  except AttributeError: pass
  else: raise Exception_expected

def run():
  exercise_journal()
  exercise_null()
  exercise_memoize()
  exercise_injector()
  exercise_memoize_with_injector()

if __name__ == '__main__':
  run()


# Define your item pipelines here
# Don't forget to add your pipeline to the ITEM_PIPELINES setting

class Craigslist4221306211ScraperPipeline(object):
    def process_item(self, item, spider):