Skip to content

GitLab

Unverified Commit bccfa9ff authored by Zhenhua Han's avatar Zhenhua Han Committed by GitHub
Browse files

execute removeGpuReservation first to release GPUs properly (#3941)

parent 30df69bf
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
ts/nni_manager/training_service/reusable/trialDispatcher.ts
View file @ bccfa9ff
...@@ -732,6 +732,9 @@ class TrialDispatcher implements TrainingService { ...@@ -732,6 +732,9 @@ class TrialDispatcher implements TrainingService {
* @param trial * @param trial
*/ */
private releaseEnvironment(trial: TrialDetail): void { private releaseEnvironment(trial: TrialDetail): void {
if (true === this.enableGpuScheduler) {
this.gpuScheduler.removeGpuReservation(trial);
}
if (trial.environment !== undefined) { if (trial.environment !== undefined) {
if (trial.environment.runningTrialCount <= 0) { if (trial.environment.runningTrialCount <= 0) {
throw new Error(`TrialDispatcher: environment ${trial.environment.id} has no counted running trial!`); throw new Error(`TrialDispatcher: environment ${trial.environment.id} has no counted running trial!`);
...@@ -740,9 +743,6 @@ class TrialDispatcher implements TrainingService { ...@@ -740,9 +743,6 @@ class TrialDispatcher implements TrainingService {
trial.environment.latestTrialReleasedTime = Date.now(); trial.environment.latestTrialReleasedTime = Date.now();
trial.environment = undefined; trial.environment = undefined;
} }
if (true === this.enableGpuScheduler) {
this.gpuScheduler.removeGpuReservation(trial);
}
} }
private async handleMetricData(trialId: string, data: any): Promise<void> { private async handleMetricData(trialId: string, data: any): Promise<void> {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment