[config] Ensure that id2label always takes precedence over num_labels

Fixes bug reported in https://github.com/huggingface/transformers/issues/4669

See #3967 for context

src/transformers/configuration_utils.py

@@ -89,6 +89,7 @@ class PretrainedConfig(object):
         self.id2label = kwargs.pop("id2label", None)
         self.label2id = kwargs.pop("label2id", None)
         if self.id2label is not None:
+            del kwargs["num_labels"]
             self.id2label = dict((int(key), value) for key, value in self.id2label.items())
             # Keys are always strings in JSON so convert ids to int here.
         else: