cuda11.1.1.dockerfile 5.3 KB
Newer Older
1
2
FROM nvcr.io/nvidia/pytorch:20.12-py3

3
4
# OS:
#   - Ubuntu: 20.04
5
#   - OpenMPI: 4.0.5
6
#   - Docker Client: 20.10.8
7
8
9
# NVIDIA:
#   - CUDA: 11.1.1
#   - cuDNN: 8.0.5
10
#   - NCCL: v2.10.3-1
11
12
13
14
# Mellanox:
#   - OFED: 5.2-2.2.3.0
#   - HPC-X: v2.8.3
#   - NCCL RDMA SHARP plugins: 7cccbc1
15
# Intel:
16
#   - mlc: v3.12
17

18
19
LABEL maintainer="SuperBench"

20
ENV DEBIAN_FRONTEND=noninteractive
21
22
RUN apt-get update && \
    apt-get install -y --no-install-recommends \
23
24
    autoconf \
    automake \
25
    bc \
26
    build-essential \
27
    ca-certificates \
28
29
    curl \
    dmidecode \
30
    ffmpeg \
31
    git \
32
    iproute2 \
33
34
    jq \
    libaio-dev \
35
36
37
    libavcodec-dev \
    libavformat-dev \
    libavutil-dev \
38
    libcap2 \
39
    libnuma-dev \
40
    libpci-dev \
41
    libswresample-dev \
42
    libtinfo5 \
43
    libtool \
44
    lshw \
45
    python3-mpi4py \
46
47
48
49
    net-tools \
    openssh-client \
    openssh-server \
    pciutils \
50
    sudo \
51
52
53
    util-linux \
    vim \
    wget \
54
    software-properties-common \
55
    && \
56
57
58
59
60
    add-apt-repository -y ppa:longsleep/golang-backports && \
    apt-get update && \
    apt-get install -y golang-1.24-go=1.24* && \
    update-alternatives --install /usr/bin/go go /usr/lib/go-1.24/bin/go 100 && \
    update-alternatives --install /usr/bin/gofmt gofmt /usr/lib/go-1.24/bin/gofmt 100 && \
61
62
63
    apt-get autoremove && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/* /tmp/* /opt/cmake-3.14.6-Linux-x86_64
64

65
66
ARG NUM_MAKE_JOBS=

67
68
69
# Install Docker
ENV DOCKER_VERSION=20.10.8
RUN cd /tmp && \
70
    wget -q https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz -O docker.tgz && \
71
72
73
    tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
    rm docker.tgz

74
# Update system config
75
76
77
78
RUN mkdir -p /root/.ssh && \
    touch /root/.ssh/authorized_keys && \
    mkdir -p /var/run/sshd && \
    sed -i "s/[# ]*PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config && \
79
    sed -i "s/[# ]*PermitUserEnvironment no/PermitUserEnvironment yes/" /etc/ssh/sshd_config && \
80
    sed -i "s/[# ]*Port.*/Port 22/" /etc/ssh/sshd_config && \
81
82
    echo "* soft nofile 1048576\n* hard nofile 1048576" >> /etc/security/limits.conf && \
    echo "root soft nofile 1048576\nroot hard nofile 1048576" >> /etc/security/limits.conf
83
84

# Install OFED
85
ENV OFED_VERSION=5.2-2.2.3.0
86
87
88
89
90
91
RUN cd /tmp && \
    wget -q http://content.mellanox.com/ofed/MLNX_OFED-${OFED_VERSION}/MLNX_OFED_LINUX-${OFED_VERSION}-ubuntu20.04-x86_64.tgz && \
    tar xzf MLNX_OFED_LINUX-${OFED_VERSION}-ubuntu20.04-x86_64.tgz && \
    MLNX_OFED_LINUX-${OFED_VERSION}-ubuntu20.04-x86_64/mlnxofedinstall --user-space-only --without-fw-update --force --all && \
    rm -rf /tmp/MLNX_OFED_LINUX-${OFED_VERSION}*

92
# Install HPC-X
93
ENV HPCX_VERSION=v2.9.0
94
RUN cd /opt && \
95
96
97
98
99
    rm -rf hpcx && \
    wget -q https://content.mellanox.com/hpc/hpc-x/${HPCX_VERSION}/hpcx-${HPCX_VERSION}-gcc-inbox-ubuntu20.04-x86_64.tbz -O hpcx.tbz && \
    tar xf hpcx.tbz && \
    mv hpcx-${HPCX_VERSION}-gcc-inbox-ubuntu20.04-x86_64 hpcx && \
    rm hpcx.tbz
100
101
102
103
104
105
106
107

# Install NCCL RDMA SHARP plugins
RUN cd /tmp && \
    git clone https://github.com/Mellanox/nccl-rdma-sharp-plugins.git && \
    cd nccl-rdma-sharp-plugins && \
    git reset --hard 7cccbc1 && \
    ./autogen.sh && \
    ./configure --prefix=/usr/local --with-cuda=/usr/local/cuda && \
108
    make -j ${NUM_MAKE_JOBS} && \
109
110
111
112
113
114
    make install && \
    cd /tmp && \
    rm -rf nccl-rdma-sharp-plugins

# Install NCCL patch
RUN cd /tmp && \
115
    git clone -b v2.10.3-1 https://github.com/NVIDIA/nccl.git && \
116
    cd nccl && \
117
    make -j ${NUM_MAKE_JOBS} src.build && \
118
119
120
121
    make install && \
    cd /tmp && \
    rm -rf nccl

122
123
# Install Intel MLC
RUN cd /tmp && \
124
    wget -q https://downloadmirror.intel.com/866182/mlc_v3.12.tgz -O mlc.tgz && \
125
    tar xzf mlc.tgz Linux/mlc && \
126
    cp ./Linux/mlc /usr/local/bin/ && \
127
    rm -rf ./Linux mlc.tgz
128

129
ENV PATH="${PATH}" \
130
    LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" \
131
132
133
134
    SB_HOME=/opt/superbench \
    SB_MICRO_PATH=/opt/superbench \
    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
135

136
137
138
139
RUN echo PATH="$PATH" > /etc/environment && \
    echo LD_LIBRARY_PATH="$LD_LIBRARY_PATH" >> /etc/environment && \
    echo SB_MICRO_PATH="$SB_MICRO_PATH" >> /etc/environment

rafsalas19's avatar
rafsalas19 committed
140
141
142
143
144
145
# Install AOCC compiler
RUN cd /tmp && \
    wget https://download.amd.com/developer/eula/aocc-compiler/aocc-compiler-4.0.0_1_amd64.deb && \
    apt install -y ./aocc-compiler-4.0.0_1_amd64.deb && \
    rm -rf aocc-compiler-4.0.0_1_amd64.deb

rafsalas19's avatar
rafsalas19 committed
146
147
148
149
150
151
152
# Install AMD BLIS
RUN cd /tmp && \
    wget https://download.amd.com/developer/eula/blis/blis-4-0/aocl-blis-linux-aocc-4.0.tar.gz && \
    tar xzf aocl-blis-linux-aocc-4.0.tar.gz && \
    mv amd-blis /opt/AMD && \
    rm -rf aocl-blis-linux-aocc-4.0.tar.gz

Yifan Xiong's avatar
Yifan Xiong committed
153
154
155
# Add config files
ADD dockerfile/etc /opt/microsoft/

156
WORKDIR ${SB_HOME}
157

158
ADD third_party third_party
159
160
161
162
163
164
# Install Rust temporarily for wandb build (required by megatron_lm target), then remove
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
    . /root/.cargo/env && \
    make -C third_party cuda -o nvbandwidth && \
    rustup self uninstall -y && \
    rm -rf /root/.cargo /root/.rustup
165
166

ADD . .
167
RUN python3 -m pip install --upgrade setuptools==65.7 importlib_metadata==6.8.0 && \
168
    python3 -m pip install --no-cache-dir .[nvworker] && \
169
    make cppbuild && \
170
171
    make postinstall && \
    rm -rf .git