"packaging/vscode:/vscode.git/clone" did not exist on "a78d0d83d0a499fe8480d7a9f493676e746c4699"
Unverified Commit 1bd3637f authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Fix EMFILE error in local training service (#1189)

parent 26792dfb
...@@ -355,7 +355,8 @@ class LocalTrainingService implements TrainingService { ...@@ -355,7 +355,8 @@ class LocalTrainingService implements TrainingService {
this.log.info('Stopping local machine training service...'); this.log.info('Stopping local machine training service...');
this.stopping = true; this.stopping = true;
for (const stream of this.jobStreamMap.values()) { for (const stream of this.jobStreamMap.values()) {
stream.destroy(); stream.end(0)
stream.emit('end')
} }
if (this.gpuScheduler !== undefined) { if (this.gpuScheduler !== undefined) {
await this.gpuScheduler.stop(); await this.gpuScheduler.stop();
...@@ -372,7 +373,9 @@ class LocalTrainingService implements TrainingService { ...@@ -372,7 +373,9 @@ class LocalTrainingService implements TrainingService {
if (stream === undefined) { if (stream === undefined) {
throw new Error(`Could not find stream in trial ${trialJob.id}`); throw new Error(`Could not find stream in trial ${trialJob.id}`);
} }
stream.destroy(); //Refer https://github.com/Juul/tail-stream/issues/20
stream.end(0)
stream.emit('end')
this.jobStreamMap.delete(trialJob.id); this.jobStreamMap.delete(trialJob.id);
} }
} }
...@@ -567,7 +570,6 @@ class LocalTrainingService implements TrainingService { ...@@ -567,7 +570,6 @@ class LocalTrainingService implements TrainingService {
buffer = remain; buffer = remain;
} }
}); });
this.jobStreamMap.set(trialJobDetail.id, stream); this.jobStreamMap.set(trialJobDetail.id, stream);
} }
......
declare module 'tail-stream' { declare module 'tail-stream' {
export interface Stream { export interface Stream {
on(type: 'data', callback: (data: Buffer) => void): void; on(type: 'data', callback: (data: Buffer) => void): void;
destroy(): void; end(data: number): void;
emit(data: string): void;
} }
export function createReadStream(path: string): Stream; export function createReadStream(path: string): Stream;
} }
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment