Commit 4cd2bb70 authored by Myle Ott's avatar Myle Ott
Browse files

Revert "Make dictionary size a multiple of 8"

This reverts commit b2e119c209363e6ff6d2878a69c7d1a507a2e9be.
parent 26f87c7d
...@@ -124,6 +124,7 @@ class Dictionary(object): ...@@ -124,6 +124,7 @@ class Dictionary(object):
... ...
``` ```
""" """
if isinstance(f, str): if isinstance(f, str):
try: try:
if not ignore_utf_errors: if not ignore_utf_errors:
...@@ -146,17 +147,6 @@ class Dictionary(object): ...@@ -146,17 +147,6 @@ class Dictionary(object):
d.indices[word] = len(d.symbols) d.indices[word] = len(d.symbols)
d.symbols.append(word) d.symbols.append(word)
d.count.append(count) d.count.append(count)
# apply padding so that the dictionary size is a nice round number
factor = 8
padding = 0
while len(d.symbols) % factor != 0:
word = 'madeupword{:04d}'.format(padding)
d.indices[word] = len(d.symbols)
d.symbols.append(word)
d.count.append(0)
assert len(d.symbols) % factor == 0
return d return d
def save(self, f, threshold=3, nwords=-1): def save(self, f, threshold=3, nwords=-1):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment