Unverified Commit 239ace15 authored by Xiaoli Wang's avatar Xiaoli Wang Committed by GitHub
Browse files

Fix TypeError: Object of type int64 is not JSON serializable (#24340)

* Fix TypeError: Object of type int64 is not JSON serializable

* Convert numpy.float64 and numpy.int64 to float and int for json serialization

* Black reformatted examples/pytorch/token-classification/run_ner_no_trainer.py

* * make style
parent ac19871c
......@@ -516,7 +516,7 @@ class Data2VecAudioModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Tes
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -373,12 +373,12 @@ class EncodecModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
uniform_init_parms = ["conv"]
ignore_init = ["lstm"]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
)
elif not any([x in name for x in ignore_init]):
elif not any(x in name for x in ignore_init):
self.assertIn(
((param.data.mean() * 1e9).round() / 1e9).item(),
[0.0, 1.0],
......
......@@ -768,7 +768,7 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
)
self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
self.assertTrue(
all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))])
all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs)))
) # token_type_ids should change output
@slow
......
......@@ -571,7 +571,7 @@ class GPTJModelLanguageGenerationTest(unittest.TestCase):
self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
self.assertTrue(
all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))])
all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs)))
) # token_type_ids should change output
@slow
......
......@@ -423,7 +423,7 @@ class HubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -684,7 +684,7 @@ class HubertRobustModelTest(ModelTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -386,7 +386,7 @@ class MCTCTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -533,7 +533,7 @@ class MCTCTRobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -334,7 +334,7 @@ class RwkvModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin
if param.requires_grad:
# check if it's a ones like
self.assertTrue(torch.allclose(param.data, torch.ones_like(param.data), atol=1e-5, rtol=1e-5))
elif any([x in name for x in ["time_mix_key", "time_mix_receptance"]]):
elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]):
if param.requires_grad:
self.assertInterval(
param.data,
......
......@@ -417,7 +417,7 @@ class SEWModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -431,7 +431,7 @@ class SEWDModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -583,7 +583,7 @@ class SpeechT5ForSpeechToTextTest(ModelTesterMixin, unittest.TestCase):
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -927,7 +927,7 @@ class SpeechT5ForTextToSpeechTest(ModelTesterMixin, unittest.TestCase):
"conv.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -1337,7 +1337,7 @@ class SpeechT5ForSpeechToSpeechTest(ModelTesterMixin, unittest.TestCase):
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -432,7 +432,7 @@ class UniSpeechRobustModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.T
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -484,7 +484,7 @@ class UniSpeechSatModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Test
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -695,7 +695,7 @@ class UniSpeechSatRobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -464,7 +464,7 @@ class FlaxWav2Vec2UtilsTest(unittest.TestCase):
negative_indices = _sample_negative_indices(features.shape, num_negatives, attention_mask=attention_mask)
# make sure that no padding tokens are sampled
self.assertTrue(all([idx not in negative_indices for idx in forbidden_indices]))
self.assertTrue(all(idx not in negative_indices for idx in forbidden_indices))
features = features.reshape(-1, hidden_size) # BTC => (BxT)C
# take negative vectors from sampled indices
......
......@@ -637,7 +637,7 @@ class Wav2Vec2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......@@ -971,7 +971,7 @@ class Wav2Vec2RobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -569,7 +569,7 @@ class Wav2Vec2ConformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -438,7 +438,7 @@ class WavLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
......
......@@ -1535,7 +1535,7 @@ class WhisperModelIntegrationTests(unittest.TestCase):
text = processor.decode(output[0])
self.assertTrue(prompt in text)
self.assertTrue(all([token in text for token in expected_tokens]))
self.assertTrue(all(token in text for token in expected_tokens))
@slow
def test_generate_with_prompt_ids_and_no_non_prompt_forced_decoder_ids(self):
......
......@@ -145,7 +145,7 @@ class OnnxExportTestCase(unittest.TestCase):
# Assert all variables are present
self.assertEqual(len(shapes), len(variable_names))
self.assertTrue(all([var_name in shapes for var_name in variable_names]))
self.assertTrue(all(var_name in shapes for var_name in variable_names))
self.assertSequenceEqual(variable_names[:3], input_vars)
self.assertSequenceEqual(variable_names[3:], output_vars)
......
......@@ -1566,7 +1566,7 @@ class TFModelTesterMixin:
return_labels=True if "labels" in inspect.signature(model_class.call).parameters.keys() else False,
)
if not any(
[tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor)]
tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor)
):
return # No integer inputs means no need for this test
......
......@@ -79,7 +79,7 @@ SMALL_TRAINING_CORPUS = [
def filter_non_english(_, pretrained_name: str):
"""Filter all the model for non-english language"""
return not any([lang in pretrained_name for lang in NON_ENGLISH_TAGS])
return not any(lang in pretrained_name for lang in NON_ENGLISH_TAGS)
def filter_roberta_detectors(_, pretrained_name: str):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment