Setup Python env
sudo apt update
sudo add-apt-repository -y ppa:deadsnakes/ppa
sudo apt install -y \
python3.10 \
python3.10-venv
python3.10 -m venv ~/.venv
source ~/.venv/bin/activate
Convert to Llama.cpp
pip install \
torch \
git+https://github.com/huggingface/transformers.git \
git+https://github.com/huggingface/peft.git \
sentencepiece
git clone https://github.com/ggerganov/llama.cpp
make
git clone https://huggingface.co/huggyllama/llama-30b huggyllama/llama-30b
find huggyllama/llama-30b/ | grep safetensors | xargs rm
git clone https://huggingface.co/IlyaGusev/saiga_30b_lora IlyaGusev/saiga_30b_lora
python ~/scripts/convert_pth.py \
huggyllama/llama-30b \
IlyaGusev/saiga_30b_lora \
saiga_30b
python ~/src/llama.cpp/convert.py \
--outtype f16 \
--vocab-dir IlyaGusev/saiga_30b_lora \
saiga_30b
rm saiga_30b/consolidated.00.pth
~/src/llama.cpp/quantize \
saiga_30b/ggml-model-f16.bin \
saiga_30b/ggml-model-q4_1.bin \
3