AMD GPU Inference(github.com)

    # install dependencies
    sudo apt -y update
    sudo apt -y upgrade
    sudo apt -y install git wget hipcc libhipblas-dev librocblas-dev cmake build-essential

    # ensure you have permissions by adding yourself to the video and render groups
    sudo usermod -aG video,render $USER
    # log out and then log back in to apply the group changes
    # you can run `rocminfo` and look for your GPU in the output to check everything is working thus far

    # download a model, build llama.cpp, and run it
    wget https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf?download=true -O dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
    git clone https://github.com/ggerganov/llama.cpp.git
    cd llama.cpp
    git checkout b3267
    HIPCXX=clang-17 cmake -H. -Bbuild -DGGML_HIPBLAS=ON -DCMAKE_HIP_ARCHITECTURES="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1100;gfx1101;gfx1102" -DCMAKE_BUILD_TYPE=Release
    make -j16 -C build
    build/bin/llama-cli -ngl 32 --color -c 2048 --temp 0.7 --repeat_penalty 1.1 -n -1 -m ../dolphin-2.2.1-mistral-7b.Q5_K_M.gguf --prompt "Once upon a time"

     hardware.graphics.enable = true;

     services.ollama = {
     enable = true;
     acceleration = "rocm";
     environmentVariables = {
       ROC_ENABLE_PRE_VEGA = "1";
       HSA_OVERRIDE_GFX_VERSION = "11.0.0";
     };
   };

  sudo apt install rocm

  Summary:
    Upgrading: 0, Installing: 203, Removing: 0, Not Upgrading: 0
    Download size: 2,369 MB / 2,371 MB
    Space needed: 35.7 GB / 822 GB available