Fix hashing for deduplication (#17048)

db034660 · Thomas Wang · GitHub · 39f8eafc · db034660
Unverified Commit db034660 authored May 04, 2022 by Thomas Wang Committed by GitHub May 04, 2022
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 1 deletion

examples/research_projects/codeparrot/scripts/preprocessing.py ...les/research_projects/codeparrot/scripts/preprocessing.py +2 -1

No files found.
--- a/examples/research_projects/codeparrot/scripts/preprocessing.py
+++ b/examples/research_projects/codeparrot/scripts/preprocessing.py
 import gzip
+import hashlib
 import multiprocessing
 import os
 import shutil
@@ -13,7 +14,7 @@ from transformers import HfArgumentParser

 def get_hash(example):
    """Get hash of content field."""
-    return {"hash": hash(example["content"])}
+    return {"hash": hashlib.md5(example["content"].strip().encode("utf-8")).hexdigest()}


 def line_stats(example):