Dockerfile.gptq

# syntax=docker/dockerfile:1

FROM nvidia/cuda:12.2.0-devel-ubuntu22.04
RUN apt-get update \
    && apt-get install -y --no-install-recommends g++ python3-dev python3-pip curl \
    && rm -rf /var/lib/apt/lists/*
WORKDIR /app
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
COPY requirements.txt requirements.txt
# Fixes No such file or directory: 'maturin'
RUN pip3 install maturin
RUN pip3 install -r requirements.txt
# Fixes exllama/cuda_ext.py:82: UserWarning: Failed to initialize NumPy: No module named 'numpy'
RUN pip3 install numpy
# https://github.com/marella/ctransformers#gptq
RUN pip3 install ctransformers[gptq]
COPY . .
EXPOSE 8000
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]