Model versioning (#8324)

* fix typo

* rm use_cdn & references, and implement new hf_bucket_url

* I'm pretty sure we don't need to `read` this file

* same here

* [BIG] file_utils.networking: do not gobble up errors anymore

* Fix CI 😇



* Apply suggestions from code review
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

* Tiny doc tweak

* Add doc + pass kwarg everywhere

* Add more tests and explain

cc @sshleifer let me know if better
Co-Authored-By: Sam Shleifer <sshleifer@gmail.com>

* Also implement revision in pipelines

In the case where we're passing a task name or a string model identifier

* Fix CI 😇



* Fix CI

* [hf_api] new methods + command line implem

* make style

* Final endpoints post-migration

* Fix post-migration

* Py3.6 compat

cc @stefan-it

Thank you @stas00
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
Co-authored-by: Sam Shleifer <sshleifer@gmail.com>

tests/test_hf_api.py

@@ -20,7 +20,7 @@ import unittest
 
 import requests
 from requests.exceptions import HTTPError
-from transformers.hf_api import HfApi, HfFolder, ModelInfo, PresignedUrl, S3Obj
+from transformers.hf_api import HfApi, HfFolder, ModelInfo, PresignedUrl, RepoObj, S3Obj
 
 
 USER = "__DUMMY_TRANSFORMERS_USER__"
@@ -35,6 +35,7 @@ FILES = [
         os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/empty.txt"),
     ),
 ]
+REPO_NAME = "my-model-{}".format(int(time.time()))
 ENDPOINT_STAGING = "https://moon-staging.huggingface.co"
 
 
@@ -78,15 +79,6 @@ class HfApiEndpointsTest(HfApiCommonTest):
         urls = self._api.presign(token=self._token, filename="nested/valid_org.txt", organization="valid_org")
         self.assertIsInstance(urls, PresignedUrl)
 
-    def test_presign_invalid(self):
-        try:
-            _ = self._api.presign(token=self._token, filename="non_nested.json")
-        except HTTPError as e:
-            self.assertIsNotNone(e.response.text)
-            self.assertTrue("Filename invalid" in e.response.text)
-        else:
-            self.fail("Expected an exception")
-
     def test_presign(self):
         for FILE_KEY, FILE_PATH in FILES:
             urls = self._api.presign(token=self._token, filename=FILE_KEY)
@@ -109,6 +101,17 @@ class HfApiEndpointsTest(HfApiCommonTest):
             o = objs[-1]
             self.assertIsInstance(o, S3Obj)
 
+    def test_list_repos_objs(self):
+        objs = self._api.list_repos_objs(token=self._token)
+        self.assertIsInstance(objs, list)
+        if len(objs) > 0:
+            o = objs[-1]
+            self.assertIsInstance(o, RepoObj)
+
+    def test_create_and_delete_repo(self):
+        self._api.create_repo(token=self._token, name=REPO_NAME)
+        self._api.delete_repo(token=self._token, name=REPO_NAME)
+
 
 class HfApiPublicTest(unittest.TestCase):
     def test_staging_model_list(self):

tests/test_modeling_tf_bert.py

@@ -323,7 +323,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
     def test_custom_load_tf_weights(self):
         model, output_loading_info = TFBertForTokenClassification.from_pretrained(
-            "jplu/tiny-tf-bert-random", use_cdn=False, output_loading_info=True
+            "jplu/tiny-tf-bert-random", output_loading_info=True
         )
         self.assertEqual(sorted(output_loading_info["unexpected_keys"]), ["mlm___cls", "nsp___cls"])
         for layer in output_loading_info["missing_keys"]:

utils/check_dummies.py

@@ -165,7 +165,7 @@ DUMMY_FUNCTION = {
 
 
 def read_init():
-    """ Read the init and exctracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects. """
+    """ Read the init and extracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects. """
     with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r", encoding="utf-8") as f:
         lines = f.readlines()