Init GPU Server

ssh root@HOST 'set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
apt-get -o DPkg::Lock::Timeout=600 -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold update
apt-get -o DPkg::Lock::Timeout=600 -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold install -y parted mergerfs

MODELS_DISK=/dev/nvme1n1
LORA_DISK=/dev/nvme0n1

for disk in "$MODELS_DISK" "$LORA_DISK"; do
  test -b "$disk"
  if lsblk -nrpo NAME "$disk" | tail -n +2 | grep -q .; then
    echo "Refusing to repartition $disk because it already has child block devices" >&2
    lsblk "$disk" >&2
    exit 1
  fi
done

parted -s "$MODELS_DISK" mklabel gpt mkpart primary ext4 0% 100%
parted -s "$LORA_DISK" mklabel gpt mkpart primary ext4 0% 100%
partprobe "$MODELS_DISK" "$LORA_DISK" || true
udevadm settle

mkfs.ext4 -F -L models "${MODELS_DISK}p1"
mkfs.ext4 -F -L loraModels "${LORA_DISK}p1"
'

ssh root@HOST 'set -euo pipefail
mkdir -p /mnt/models /mnt/loraModels /mnt/official-models

MODEL_PART=/dev/nvme1n1p1
LORA_PART=/dev/nvme0n1p1
MODEL_UUID=$(blkid -s UUID -o value "$MODEL_PART")
LORA_UUID=$(blkid -s UUID -o value "$LORA_PART")

cp /etc/fstab "/etc/fstab.drawthings.$(date +%Y%m%d%H%M%S).bak"
grep -vE "/mnt/models|/mnt/loraModels|/mnt/official-models|/mnt/official_models" /etc/fstab > /etc/fstab.drawthings.new
mv /etc/fstab.drawthings.new /etc/fstab

{
  echo "UUID=$MODEL_UUID /mnt/models ext4 defaults,nofail 0 2"
  echo "UUID=$LORA_UUID /mnt/loraModels ext4 defaults,nofail 0 2"
  echo "/mnt/models/official-models:/mnt/loraModels/models_extra /mnt/official-models fuse.mergerfs allow_other,use_ino,ro,nofail 0 0"
} >> /etc/fstab

if grep -q "^#user_allow_other" /etc/fuse.conf; then
  sed -i "s/^#user_allow_other/user_allow_other/" /etc/fuse.conf
elif ! grep -q "^user_allow_other" /etc/fuse.conf; then
  echo user_allow_other >> /etc/fuse.conf
fi

mountpoint -q /mnt/official-models && umount /mnt/official-models || true
mountpoint -q /mnt/models || mount /mnt/models
mountpoint -q /mnt/loraModels || mount /mnt/loraModels
mkdir -p /mnt/models/official-models /mnt/loraModels/models_extra /mnt/official-models
mountpoint -q /mnt/official-models || mount /mnt/official-models

lsblk -o NAME,SIZE,TYPE,MOUNTPOINTS,FSTYPE,UUID
df -h | grep -E "Filesystem|/mnt/models|/mnt/loraModels|/mnt/official-models"
'

ssh root@HOST 'set -e
echo "=== GPUs ==="
nvidia-smi --query-gpu=name,driver_version --format=csv,noheader
echo "=== CUDA ==="
/usr/local/cuda/bin/nvcc --version
echo "=== NVIDIA Container Toolkit ==="
nvidia-ctk --version
echo "=== Docker ==="
docker --version
docker info --format "Runtimes={{json .Runtimes}} Default={{.DefaultRuntime}}"
echo "=== Draw Things image ==="
docker image inspect drawthingsai/draw-things-grpc-server-cli:latest --format "{{.Id}} {{.RepoTags}}"
echo "=== Mounts ==="
lsblk -o NAME,SIZE,TYPE,MOUNTPOINTS,FSTYPE
df -h | grep -E "Filesystem|/mnt/models|/mnt/loraModels|/mnt/official-models"
'

ssh root@HOST 'docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi'

nvme1n1p1 /mnt/models     ext4
nvme0n1p1 /mnt/loraModels ext4

ssh root@HOST 'test -f /var/run/reboot-required && cat /var/run/reboot-required || echo no'