conda create -n Torch python=3.10
conda activate Torch
pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1

# on CentOS 7
sudo yum install -y centos-release-scl-rh
sudo yum update
sudo yum install -y devtoolset-9
source /opt/rh/devtoolset-9/enable

# on debian 
sudo apt update
sudo apt install build-essential manpages-dev

git clone https://github.com/microsoft/DeepSpeed.git

cd DeepSpeed

git checkout v0.9.5

# !!! below is single command
TORCH_CUDA_ARCH_LIST="6.1;7.5;8.6;8.9" \
DS_BUILD_CCL_COMM=1 \
DS_BUILD_CPU_ADAM=1 \
DS_BUILD_CPU_ADAGRAD=1 \
DS_BUILD_FUSED_ADAM=1 \
DS_BUILD_FUSED_LAMB=1 \
DS_BUILD_UTILS=1 \
python setup.py build_ext -j24 bdist_wheel

pip install ./dist/deepspeed-0.9.5+fc9e1ee-*-linux_x86_64.whl

git clone https://github.com/OvJat/DeepSpeedTutorial.git
cd DeepSpeedTutorial
# editor hostfile 
./run_dist.sh &
disown 
# check logging
tail -f ./logs/run_dist.sh.log