"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "13936a962102ed20424838fe5d445a28b5225d08"
Unverified Commit 88b3a91e authored by Lalit Pagaria's avatar Lalit Pagaria Committed by GitHub
Browse files

Handle the case when title is None (#7941)

parent 023f0f37
...@@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict: ...@@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
"""Split documents into passages""" """Split documents into passages"""
titles, texts = [], [] titles, texts = [], []
for title, text in zip(documents["title"], documents["text"]): for title, text in zip(documents["title"], documents["text"]):
for passage in split_text(text): if text is not None:
titles.append(title) for passage in split_text(text):
texts.append(passage) titles.append(title if title is not None else "")
texts.append(passage)
return {"title": titles, "text": texts} return {"title": titles, "text": texts}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment