for dependency in hard_dependencies:
    try:
        __import__(dependency)
    except ImportError as e:
        missing_dependencies.append(dependency)

# pylint: disable-msg=W0614,W0401,W0611,W0622

# flake8: noqa

__docformat__ = 'restructuredtext'

# Let users know if they're missing any of our hard dependencies
hard_dependencies = ("numpy", "pytz", "dateutil")
missing_dependencies = []

if missing_dependencies:
    raise ImportError(
        "Missing required dependencies {0}".format(missing_dependencies))
del hard_dependencies, dependency, missing_dependencies

# numpy compat
from pandas.compat.numpy import *

try:
    from pandas._libs import (hashtable as _hashtable,
                             lib as _lib,
                             tslib as _tslib)
except ImportError as e:  # pragma: no cover
    # hack but overkill to use re
    module = str(e).replace('cannot import name ', '')
    raise ImportError("C extension: {0} not built. If you want to import "
                      "pandas from the source directory, you may need to run "
                      "'python setup.py build_ext --inplace --force' to build "
                      "the C extensions first.".format(module))

from datetime import datetime

# let init-time option registration happen
import pandas.core.config_init

from pandas.core.api import *
from pandas.core.sparse.api import *
from pandas.tseries.api import *
from pandas.core.computation.api import *
from pandas.core.reshape.api import *

from pandas.util._print_versions import show_versions
from pandas.io.api import *
from pandas.util._tester import test
import pandas.testing
import pandas.arrays

# use the closest tagged version if possible
from ._version import get_versions
v = get_versions()
__version__ = v.get('closest-tag', v['version'])
__git_version__ = v.get('full-revisionid')
del get_versions, v

# module level doc-string
__doc__ = """
pandas - a powerful data analysis and manipulation library for Python
=====================================================================

**pandas** is a Python package providing fast, flexible, and expressive data
structures designed to make working with "relational" or "labeled" data both
easy and intuitive. It aims to be the fundamental high-level building block for
doing practical, **real world** data analysis in Python. Additionally, it has
the broader goal of becoming **the most powerful and flexible open source data
analysis / manipulation tool available in any language**. It is already well on
its way toward this goal.

Main Features
-------------
Here are just a few of the things that pandas does well:

  - Easy handling of missing data in floating point as well as non-floating
    point data.
  - Size mutability: columns can be inserted and deleted from DataFrame and
    higher dimensional objects
  - Automatic and explicit data alignment: objects can be explicitly aligned
    to a set of labels, or the user can simply ignore the labels and let
    `Series`, `DataFrame`, etc. automatically align the data for you in
    computations.
  - Powerful, flexible group by functionality to perform split-apply-combine
    operations on data sets, for both aggregating and transforming data.
  - Make it easy to convert ragged, differently-indexed data in other Python
    and NumPy data structures into DataFrame objects.
  - Intelligent label-based slicing, fancy indexing, and subsetting of large
    data sets.
  - Intuitive merging and joining data sets.
  - Flexible reshaping and pivoting of data sets.
  - Hierarchical labeling of axes (possible to have multiple labels per tick).
  - Robust IO tools for loading data from flat files (CSV and delimited),
    Excel files, databases, and saving/loading data from the ultrafast HDF5
    format.
  - Time series-specific functionality: date range generation and frequency
    conversion, moving window statistics, moving window linear regressions,
    date shifting and lagging, etc.
"""


try:
        __import__(dependency)
    except ImportError as e:
        missing_dependencies.append(dependency)

# pylint: disable-msg=W0614,W0401,W0611,W0622

# flake8: noqa

__docformat__ = 'restructuredtext'

# Let users know if they're missing any of our hard dependencies
hard_dependencies = ("numpy", "pytz", "dateutil")
missing_dependencies = []

for dependency in hard_dependencies:


# pylint: disable-msg=W0614,W0401,W0611,W0622

# flake8: noqa

__docformat__ = 'restructuredtext'

# Let users know if they're missing any of our hard dependencies
hard_dependencies = ("numpy", "pytz", "dateutil")
missing_dependencies = []

for dependency in hard_dependencies:
    try:
        __

import__(dependency)
    except ImportError as e:
        missing_dependencies.append(dependency)

if missing_dependencies:
    raise ImportError(
        "Missing required dependencies {0}".format(missing_dependencies))
del hard_dependencies, dependency, missing_dependencies

# numpy compat
from pandas.compat.numpy import *

try:
    from pandas._libs import (hashtable as _hashtable,
                             lib as _lib,
                             tslib as _tslib)
except ImportError as e:  # pragma: no cover
    # hack but overkill to use re
    module = str(e).replace('cannot import name ', '')
    raise ImportError("C extension: {0} not built. If you want to import "
                      "pandas from the source directory, you may need to run "
                      "'python setup.py build_ext --inplace --force' to build "
                      "the C extensions first.".format(module))

from datetime import datetime

# let init-time option registration happen
import pandas.core.config_init

from pandas.core.api import *
from pandas.core.sparse.api import *
from pandas.tseries.api import *
from pandas.core.computation.api import *
from pandas.core.reshape.api import *

from pandas.util._print_versions import show_versions
from pandas.io.api import *
from pandas.util._tester import test
import pandas.testing
import pandas.arrays

# use the closest tagged version if possible
from ._version import get_versions
v = get_versions()
__version__ = v.get('closest-tag', v['version'])
__git_version__ = v.get('full-revisionid')
del get_versions, v

# module level doc-string
__doc__ = """
pandas - a powerful data analysis and manipulation library for Python
=====================================================================

**pandas** is a Python package providing fast, flexible, and expressive data
structures designed to make working with "relational" or "labeled" data both
easy and intuitive. It aims to be the fundamental high-level building block for
doing practical, **real world** data analysis in Python. Additionally, it has
the broader goal of becoming **the most powerful and flexible open source data
analysis / manipulation tool available in any language**. It is already well on
its way toward this goal.

Main Features
-------------
Here are just a few of the things that pandas does well:

  - Easy handling of missing data in floating point as well as non-floating
    point data.
  - Size mutability: columns can be inserted and deleted from DataFrame and
    higher dimensional objects
  - Automatic and explicit data alignment: objects can be explicitly aligned
    to a set of labels, or the user can simply ignore the labels and let
    `Series`, `DataFrame`, etc. automatically align the data for you in
    computations.
  - Powerful, flexible group by functionality to perform split-apply-combine
    operations on data sets, for both aggregating and transforming data.
  - Make it easy to convert ragged, differently-indexed data in other Python
    and NumPy data structures into DataFrame objects.
  - Intelligent label-based slicing, fancy indexing, and subsetting of large
    data sets.
  - Intuitive merging and joining data sets.
  - Flexible reshaping and pivoting of data sets.
  - Hierarchical labeling of axes (possible to have multiple labels per tick).
  - Robust IO tools for loading data from flat files (CSV and delimited),
    Excel files, databases, and saving/loading data from the ultrafast HDF5
    format.
  - Time series-specific functionality: date range generation and frequency
    conversion, moving window statistics, moving window linear regressions,
    date shifting and lagging, etc.
"""


def _get_art_abs(self, story_file, version):

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

"""Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


"""Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


""" Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):

        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)

        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(to

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


for line in f:
                lines.append(line.strip())

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:

        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)

import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()

            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


import collections
import os
import hashlib

from paddle.dataset.common import md5file
from paddlenlp.utils.downloader import get_path_from_url, _decompress
from paddlenlp.utils.env import DATA_HOME
from paddlenlp.utils.log import logger
from . import DatasetBuilder

class CnnDailymail(DatasetBuilder):
    """
    CNN/DailyMail non-anonymized summarization dataset.
    The CNN / DailyMail Dataset is an English-language dataset containing
    just over 300k unique news articles as written by journalists at CNN
    nd the Daily Mail. The current version supports both extractive and
    abstractive summarization, though the original version was created
    for machine reading and comprehension and abstractive question answering.

    Version 1.0.0 aimed to support supervised neural methodologies for machine
    reading and question answering with a large amount of real natural language
    training data and released about 313k unique articles and nearly 1M Cloze
    style questions to go with the articles.
    Versions 2.0.0 and 3.0.0 changed the structure of the dataset to support
    summarization rather than question answering. Version 3.0.0 provided a
    non-anonymized version of the data, whereas both the previous versions were
    preprocessed to replace named entities with unique identifier labels.

    An updated version of the code that does not anonymize the data is available
    at http://example.com.
    """
    lazy = False
    META_INFO = collections.namedtuple("META_INFO", ("file", "url", "md5"))
    SPLITS = {
        "train": META_INFO(
            "all_train.txt",
            "http://example.com",
            "c8ca98cfcb6cf3f99a404552568490bc"),
        "dev": META_INFO(
            "all_val.txt",
            "http://example.com",
            "83a3c483b3ed38b1392285bed668bfee"),
        "test": META_INFO(
            "all_test.txt",
            "http://example.com",
            "4f3ac04669934dbc746b7061e68a0258")
    }
    cnn_dailymail = {
        "cnn": {
            "url":
            "http://example.com",
            "md5": "85ac23a1926a831e8f46a6b8eaf57263",
            "file_num": 92579
        },
        "dailymail": {
            "url":
            "http://example.com",
            "md5": "f9c5f565e8abe86c38bfa4ae8f96fd72",
            "file_num": 219506
        }
    }

    def _read_text_file(self, text_file):
        lines = []
        with open(text_file, "r", encoding="utf8") as f:
            for line in f:
                lines.append(line.strip())
        return lines

    def _get_url_hashes(self, path):
        """Get hashes of urls in file."""
        urls = self._read_text_file(path)

        def url_hash(u):
            h = hashlib.sha1()
            try:
                u = u.encode("utf-8")
            except UnicodeDecodeError:
                logger.error("Cannot hash url: %s", u)
            h.update(u)
            return h.hexdigest()

        return {url_hash(u): True for u in urls}

    def _get_hash_from_path(self, p):
        """Extract hash from path."""
        basename = os.path.basename(p)
        return basename[0:basename.find(".story")]

    def _find_files(self, dl_paths, publisher, url_dict):
        """Find files corresponding

to urls."""
        if publisher == "cnn":
            top_dir = os.path.join(dl_paths["cnn"], "stories")
        elif publisher == "dailymail":
            top_dir = os.path.join(dl_paths["dailymail"], "stories")
        else:
            logger.error("Unsupported publisher: %s", publisher)
        files = sorted(os.listdir(top_dir))

        ret_files = []
        for p in files:
            if self._get_hash_from_path(p) in url_dict:
                ret_files.append(os.path.join(top_dir, p))
        return ret_files

    def _subset_filenames(self, dl_paths, split):
        """Get filenames for a particular split."""
        # Get filenames for a split.
        urls = self._get_url_hashes(dl_paths[split])
        cnn = self._find_files(dl_paths, "cnn", urls)
        dm = self._find_files(dl_paths, "dailymail", urls)
        return cnn + dm

    def _get_art_abs(self, story_file, version):
        """Get abstract (highlights) and article from a story file path."""
        # Based on http://example.com
        #     make_datafiles.py

        lines = self._read_text_file(story_file)

        # The github code lowercase the text and we removed it in 3.0.0.

        # Put periods on the ends of lines that are missing them
        # (this is a problem in the dataset because many image captions don't end in
        # periods; consequently they end up in the body of the article as run-on
        def fix_missing_period(line):
            """Adds a period to a line that is missing a period."""
            if "@highlight" in line:
                return line
            if not line:
                return line
            if line[-1] in [
                    ".", "!", "?", "...", "'", "`", '"', "\u2019", "\u201d", ")"
            ]:
                return line
            return line + " ."

        lines = [fix_missing_period(line) for line in lines]

        # Separate out article and abstract sentences
        article_lines = []
        highlights = []
        next_is_highlight = False
        for line in lines:
            if not line:
                continue  # empty line
            elif line.startswith("@highlight"):
                next_is_highlight = True
            elif next_is_highlight:
                highlights.append(line)
            else:
                article_lines.append(line)

        # Make article into a single string
        article = " ".join(article_lines)

        if version >= "2.0.0":
            abstract = "\n".join(highlights)
        else:
            abstract = " ".join(highlights)

        return article, abstract

    def _get_data(self, mode):
        """ Check and download Dataset """
        dl_paths = {}
        version = self.config.get("version", "3.0.0")
        if version not in ["1.0.0", "2.0.0", "3.0.0"]:
            raise ValueError("Unsupported version: %s" % version)
        dl_paths["version"] = version
        default_root = os.path.join(DATA_HOME, self.__class__.__name__)
        for k, v in self.cnn_dailymail.items():
            dir_path = os.path.join(default_root, k)
            if not os.path.exists(dir_path):
                get_path_from_url(v["url"], default_root, v["md5"])
            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
            if file_num != v["file_num"]:
                logger.warning(
                    "Number of %s stories is %d != %d, decompress again." %
                    (k, file_num, v["file_num"]))
                _decompress(
                    os.path.join(default_root, os.path.basename(v["url"])))
            dl_paths[k] = dir_path
        filename, url, data_hash = self.SPLITS[mode]
        fullname = os.path.join(default_root, filename)
        if not os.path.exists(fullname) or (data_hash and
                                            not md5file(fullname) == data_hash):
            get_path_from_url(url, default_root, data_hash)
        dl_paths[mode] = fullname
        return dl_paths

    def _read(self, dl_paths, split):
        files = self._subset_filenames(dl_paths, split)
        for p in files:
            article, highlights = self._get_art_abs(p, dl_paths["version"])
            if not article or not highlights:
                continue
            yield {
                "article": article,
                "highlights": highlights,
                "id": self._get_hash_from_path(p),
            }


print('The principal is equal to', p)
	print('The time period is equal to', t)
	print('The rate of interest is equal to',r)

	si = (p * t * r)/100

	print('The Simple Interest is equal to', si)
	return si

simple_interest(8, 6, 8)


print('The principal is equal to', p)
	print('The time period is equal to', t)
	print('The rate of interest is equal to',r)

	si = (p * t * r)/100

	print('The Simple Interest is equal to', si)
	return si


print('The principal is equal to', p)
	print('The time period is equal to', t)

	print('The rate of interest is equal to',r)

	si = (p * t * r)/100

	print('The Simple Interest is equal to', si)
	return si

simple_interest(8, 6, 8)


print('The rate of interest is equal to',r)

	si = (p * t * r)/100

def simple_interest(p,t,r):
	print('The principal is equal to', p)
	print('The time period is equal to', t)

	print('The Simple Interest is equal to', si)
	return si

simple_interest(8, 6, 8)


print('The Simple Interest is equal to', si)
	return si

def simple_interest(p,t,r):
	print('The principal is equal to', p)
	print('The time period is equal to', t)
	print('The rate of interest is equal to',r)

	si = (p * t * r)/100


def simple_interest(p,t,r):
	print('The principal is equal to', p)
	print('The time period is equal to', t)
	print('The rate of interest is equal to',r)

	si = (p * t * r)/100

	print('The S


e principal is equal to', p)
	print('The time period is equal to', t)
	print('The rate of interest is equal to',r)

	si = (p * t * r)/100

	print('The Simple Interest is equal to', si)
	return si

simple_interest(8, 6, 8)


import json

from alipay.aop.api.constant.ParamConstants import *
from alipay.aop.api.domain.BankcardExtInfo import BankcardExtInfo

class Participant(object):

    def __init__(self):
        self._bankcard_ext_info = None
        self._ext_info = None
        self._identity = None
        self._identity_type = None
        self._merchant_user_info = None
        self._name = None

    @property
    def bankcard_ext_info(self):
        return self._bankcard_ext_info

    @bankcard_ext_info.setter
    def bankcard_ext_info(self, value):
        if isinstance(value, BankcardExtInfo):
            self._bankcard_ext_info = value
        else:
            self._bankcard_ext_info = BankcardExtInfo.from_alipay_dict(value)
    @property
    def ext_info(self):
        return self._ext_info

    @ext_info.setter
    def ext_info(self, value):
        self._ext_info = value
    @property
    def identity(self):
        return self._identity

    @identity.setter
    def identity(self, value):
        self._identity = value
    @property
    def identity_type(self):
        return self._identity_type

    @identity_type.setter
    def identity_type(self, value):
        self._identity_type = value
    @property
    def merchant_user_info(self):
        return self._merchant_user_info

    @merchant_user_info.setter
    def merchant_user_info(self, value):
        self._merchant_user_info = value
    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value

    def to_alipay_dict(self):
        params = dict()
        if self.bankcard_ext_info:
            if hasattr(self.bankcard_ext_info, 'to_alipay_dict'):
                params['bankcard_ext_info'] = self.bankcard_ext_info.to_alipay_dict()
            else:
                params['bankcard_ext_info'] = self.bankcard_ext_info
        if self.ext_info:
            if hasattr(self.ext_info, 'to_alipay_dict'):
                params['ext_info'] = self.ext_info.to_alipay_dict()
            else:
                params['ext_info'] = self.ext_info
        if self.identity:
            if hasattr(self.identity, 'to_alipay_dict'):
                params['identity'] = self.identity.to_alipay_dict()
            else:
                params['identity'] = self.identity
        if self.identity_type:
            if hasattr(self.identity_type, 'to_alipay_dict'):
                params['identity_type'] = self.identity_type.to_alipay_dict()
            else:
                params['identity_type'] = self.identity_type
        if self.merchant_user_info:
            if hasattr(self.merchant_user_info, 'to_alipay_dict'):
                params['merchant_user_info'] = self.merchant_user_info.to_alipay_dict()
            else:
                params['merchant_user_info'] = self.merchant_user_info
        if self.name:
            if hasattr(self.name, 'to_alipay_dict'):
                params['name'] = self.name.to_alipay_dict()
            else:
                params['name'] = self.name
        return params

    @staticmethod

if not d:
            return None
        o = Participant()
        if 'bankcard_ext_info' in d:
            o.bankcard_ext_info = d['bankcard_ext_info']
        if 'ext_info' in d:
            o.ext_info = d['ext_info']
        if 'identity' in d:
            o.identity = d['identity']
        if 'identity_type' in d:
            o.identity_type = d['identity_type']
        if 'merchant_user_info' in d:
            o.merchant_user_info = d['merchant_user_info']
        if 'name' in d:
            o.name = d['name']
        return o