Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
BingBertSquad_test_common.py 2.45 KB
Newer Older
Shaden Smith's avatar
Porting BingBertSquad test (#70)  
Shaden Smith committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67 
# Copyright (c) 2019, The Microsoft DeepSpeed Team. All rights reserved.
#

import unittest
import subprocess
import os
import time


class BaseTestCase(unittest.TestCase):
    def __init__(self, methodName="DeepSpeed performance test"):
        super(BaseTestCase, self).__init__(methodName)
        self.test_dir = "./test"
        self.baseline_dir = "./baseline"
        self.timestr = time.strftime("%Y%m%d-%H%M%S")

    def gen_output_name(self, test_config, prefix):
        other_args = test_config["other_args"] if "other_args" in test_config else ""
        zero_args = "_zero" if "zero" in test_config and test_config["zero"] else ""
        other_args = other_args.strip(' -\\').replace(" ", "").replace("\"", "")

        if other_args:
            other_args = "_" + other_args

        if test_config["deepspeed"]:
            file_name = "_gpu{0}_{1}_ds{2}-{3}.log".format(test_config["gpus"],
                                                           other_args,
                                                           zero_args,
                                                           self.timestr)
            save_dir = self.test_dir
        else:
            file_name = "_gpu{0}_{1}.log".format(test_config["gpus"], other_args)
            save_dir = self.baseline_dir

        return os.path.join(save_dir, prefix + file_name)

    def ensure_directory_exists(self, filename):
        dirname = os.path.dirname(filename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

    def clean_test_env(self):
        cmd = "dlts_ssh pkill -9 -f /usr/bin/python"
        print(cmd)
        subprocess.run(cmd, shell=True, check=False, executable='/bin/bash')
        time.sleep(20)

    def run_BingBertSquad_test(self, test_config, output):
        ds_flag = " -d --deepspeed_config " + test_config["json"] if test_config[
            "deepspeed"] else " "
        other_args = " " + test_config[
            "other_args"] if "other_args" in test_config else " "

        cmd = "./run_BingBertSquad_sanity.sh -e 1 -g {0} {1} {2}".format(
            test_config["gpus"],
            other_args,
            ds_flag)

        self.ensure_directory_exists(output)
        with open(output, "w") as f:
            print(cmd)
            subprocess.run(cmd,
                           shell=True,
                           check=False,
                           executable='/bin/bash',
                           stdout=f,
                           stderr=f)