Unverified Commit 88b3a91e authored by Lalit Pagaria's avatar Lalit Pagaria Committed by GitHub
Browse files

Handle the case when title is None (#7941)

parent 023f0f37
......@@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
"""Split documents into passages"""
titles, texts = [], []
for title, text in zip(documents["title"], documents["text"]):
for passage in split_text(text):
titles.append(title)
texts.append(passage)
if text is not None:
for passage in split_text(text):
titles.append(title if title is not None else "")
texts.append(passage)
return {"title": titles, "text": texts}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment