"...hubert/git@developer.sourcefind.cn:OpenDAS/torchaudio.git" did not exist on "9877f54491a7081266207e1a999dd47bc2bba17e"
Unverified Commit 1bd3637f authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Fix EMFILE error in local training service (#1189)

parent 26792dfb
......@@ -355,7 +355,8 @@ class LocalTrainingService implements TrainingService {
this.log.info('Stopping local machine training service...');
this.stopping = true;
for (const stream of this.jobStreamMap.values()) {
stream.destroy();
stream.end(0)
stream.emit('end')
}
if (this.gpuScheduler !== undefined) {
await this.gpuScheduler.stop();
......@@ -372,7 +373,9 @@ class LocalTrainingService implements TrainingService {
if (stream === undefined) {
throw new Error(`Could not find stream in trial ${trialJob.id}`);
}
stream.destroy();
//Refer https://github.com/Juul/tail-stream/issues/20
stream.end(0)
stream.emit('end')
this.jobStreamMap.delete(trialJob.id);
}
}
......@@ -567,7 +570,6 @@ class LocalTrainingService implements TrainingService {
buffer = remain;
}
});
this.jobStreamMap.set(trialJobDetail.id, stream);
}
......
declare module 'tail-stream' {
export interface Stream {
on(type: 'data', callback: (data: Buffer) => void): void;
destroy(): void;
end(data: number): void;
emit(data: string): void;
}
export function createReadStream(path: string): Stream;
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment