Dockerfile 2.11 KB
Newer Older
OlivierDehaene's avatar
OlivierDehaene committed
1
FROM rust:1.65 as router-builder
Olivier Dehaene's avatar
Olivier Dehaene committed
2
3
4

WORKDIR /usr/src

5
COPY rust-toolchain.toml rust-toolchain.toml
Olivier Dehaene's avatar
Olivier Dehaene committed
6
7
8
9
10
11
12
COPY proto proto
COPY router router

WORKDIR /usr/src/router

RUN cargo install --path .

OlivierDehaene's avatar
OlivierDehaene committed
13
FROM rust:1.65 as launcher-builder
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
14
15
16

WORKDIR /usr/src

17
COPY rust-toolchain.toml rust-toolchain.toml
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
18
19
20
21
22
23
24
COPY launcher launcher

WORKDIR /usr/src/launcher

RUN cargo install --path .

FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
Olivier Dehaene's avatar
Olivier Dehaene committed
25
26
27
28

ENV LANG=C.UTF-8 \
    LC_ALL=C.UTF-8 \
    DEBIAN_FRONTEND=noninteractive \
29
30
    MODEL_BASE_PATH=/data \
    MODEL_ID=bigscience/bloom-560m \
31
    QUANTIZE=false \
32
    NUM_GPUS=1 \
OlivierDehaene's avatar
OlivierDehaene committed
33
    SAFETENSORS_FAST_GPU=1 \
Olivier Dehaene's avatar
Olivier Dehaene committed
34
    CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
Olivier Dehaene's avatar
Olivier Dehaene committed
35
    NCCL_ASYNC_ERROR_HANDLING=1 \
Olivier Dehaene's avatar
Olivier Dehaene committed
36
37
38
39
40
41
42
    CUDA_HOME=/usr/local/cuda \
    LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \
    CONDA_DEFAULT_ENV=text-generation \
    PATH=$PATH:/opt/miniconda/envs/text-generation/bin:/opt/miniconda/bin:/usr/local/cuda/bin

SHELL ["/bin/bash", "-c"]

Nicolas Patry's avatar
Nicolas Patry committed
43
RUN apt-get update && apt-get install -y unzip curl libssl-dev && rm -rf /var/lib/apt/lists/*
Olivier Dehaene's avatar
Olivier Dehaene committed
44
45

RUN cd ~ && \
Nicolas Patry's avatar
Nicolas Patry committed
46
    curl -L -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
Olivier Dehaene's avatar
Olivier Dehaene committed
47
48
49
50
    chmod +x Miniconda3-latest-Linux-x86_64.sh && \
    bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \
    conda create -n text-generation python=3.9 -y

Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
51
52
53
54
WORKDIR /usr/src

COPY server/Makefile server/Makefile

Olivier Dehaene's avatar
Olivier Dehaene committed
55
# Install specific version of torch
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
56
RUN cd server && make install-torch
Olivier Dehaene's avatar
Olivier Dehaene committed
57
58

# Install specific version of transformers
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
59
RUN cd server && make install-transformers
Olivier Dehaene's avatar
Olivier Dehaene committed
60
61

# Install server
Nicolas Patry's avatar
Nicolas Patry committed
62
COPY proto proto
Olivier Dehaene's avatar
Olivier Dehaene committed
63
64
COPY server server
RUN cd server && \
Nicolas Patry's avatar
Nicolas Patry committed
65
    make gen-server && \
66
    /opt/miniconda/envs/text-generation/bin/pip install ".[bnb]" --no-cache-dir
Olivier Dehaene's avatar
Olivier Dehaene committed
67
68

# Install router
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
69
COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router
Nicolas Patry's avatar
Nicolas Patry committed
70
# Install launcher
Olivier Dehaene's avatar
v0.1.0  
Olivier Dehaene committed
71
COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher
Olivier Dehaene's avatar
Olivier Dehaene committed
72

73
CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS --json-output