[HPO] update to sigopt new experiment api (#18147)

* [HPO] update to sigopt new experiment api
* follow https://docs.sigopt.com/experiments

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* [HPO] use new API if sigopt version >= 8.0.0
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

src/transformers/integrations.py

@@ -305,7 +305,36 @@ def run_hp_search_ray(trainer, n_trials: int, direction: str, **kwargs) -> BestR
 
 
 def run_hp_search_sigopt(trainer, n_trials: int, direction: str, **kwargs) -> BestRun:
+    import sigopt
+    from transformers.utils.versions import importlib_metadata
 
+    if importlib_metadata.version("sigopt") >= "8.0.0":
+        sigopt.set_project("huggingface")
+
+        experiment = sigopt.create_experiment(
+            name="huggingface-tune",
+            type="offline",
+            parameters=trainer.hp_space(None),
+            metrics=[dict(name="objective", objective=direction, strategy="optimize")],
+            parallel_bandwidth=1,
+            budget=n_trials,
+        )
+
+        logger.info(f"created experiment: https://app.sigopt.com/experiment/{experiment.id}")
+
+        for run in experiment.loop():
+            with run:
+                trainer.objective = None
+                trainer.train(resume_from_checkpoint=None, trial=run.run)
+                # If there hasn't been any evaluation during the training loop.
+                if getattr(trainer, "objective", None) is None:
+                    metrics = trainer.evaluate()
+                    trainer.objective = trainer.compute_objective(metrics)
+                run.log_metric("objective", trainer.objective)
+
+        best = list(experiment.get_best_runs())[0]
+        best_run = BestRun(best.id, best.values["objective"].value, best.assignments)
+    else:
         from sigopt import Connection
 
         conn = Connection()
@@ -339,7 +368,6 @@ def run_hp_search_sigopt(trainer, n_trials: int, direction: str, **kwargs) -> Be
 
         best = list(conn.experiments(experiment.id).best_assignments().fetch().iterate_pages())[0]
         best_run = BestRun(best.id, best.value, best.assignments)
-
     return best_run