v1.0.55

allamo/dataset/dataset.py

@@ -64,7 +64,7 @@ class AllamoDataset:
         self.processed_files.append(load_dataset_file)
         new_data = None
         if load_dataset_file.endswith('.bin'):
-            assert self.training_type == 'pre', 'NumPy format is supported only for pre-training'
+            # assert self.training_type == 'pre', 'NumPy format is supported only for pre-training'
             step_size = self.world_size * self.sample_size
             new_data = torch.from_numpy(np.fromfile(load_dataset_file, dtype=np.uint16).astype(np.int16))
             if step_size > len(new_data):

inference/sample.py

@@ -68,7 +68,7 @@ class AllamoSampler:
         else:
             raise Exception('Tokenizer is not provided. Please specify either a Tiktoken tokenizer or a HuggingFace tokenizer')
         # ensure that the tokenizer and model vocabulary sizes are equal
-        assert len(tokenizer) == self.model.config.vocab_size
+        # assert len(tokenizer) == self.model.config.vocab_size
         self.tokenizer = tokenizer
     
     def tokenize_prompt(self, text: str):