From 23b4e1a8eed1b1db4f8f494e3f26e01608bf0006 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 20 Nov 2025 10:58:56 +0200 Subject: [PATCH] win amd GPU passtrough --- .../rincoin/docker-compose.yml | 6 + .../windows/GPU-PASSTHROUGH.md | 203 ++++++++++++++++++ .../windows/docker-compose.yml | 9 +- .../windows/fix-grub-iommu.sh | 43 ++++ .../windows/fix-grub-manual.sh | 48 +++++ .../windows/setup-gpu-passthrough.sh | 109 ++++++++++ .../windows/verify-gpu-passthrough.sh | 96 +++++++++ 7 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 portainer-compose-stacks/windows/GPU-PASSTHROUGH.md create mode 100644 portainer-compose-stacks/windows/fix-grub-iommu.sh create mode 100644 portainer-compose-stacks/windows/fix-grub-manual.sh create mode 100644 portainer-compose-stacks/windows/setup-gpu-passthrough.sh create mode 100644 portainer-compose-stacks/windows/verify-gpu-passthrough.sh diff --git a/portainer-compose-stacks/rincoin/docker-compose.yml b/portainer-compose-stacks/rincoin/docker-compose.yml index ce07a4d..68309ae 100644 --- a/portainer-compose-stacks/rincoin/docker-compose.yml +++ b/portainer-compose-stacks/rincoin/docker-compose.yml @@ -16,3 +16,9 @@ services: - -conf=/data/rincoin.conf - -printtoconsole + + + + + + diff --git a/portainer-compose-stacks/windows/GPU-PASSTHROUGH.md b/portainer-compose-stacks/windows/GPU-PASSTHROUGH.md new file mode 100644 index 0000000..c16d83d --- /dev/null +++ b/portainer-compose-stacks/windows/GPU-PASSTHROUGH.md @@ -0,0 +1,203 @@ +# AMD Strix Halo iGPU Passthrough to Windows Container + +This guide configures PCI passthrough for the AMD Strix Halo integrated GPU to the Windows Docker container, enabling GPU-accelerated applications. + +## Problem + +The Windows container was showing "Red Hat VirtIO GPU DOD Controller" instead of the AMD GPU because: +- IOMMU was disabled (`amd_iommu=off`) +- GPU was not passed through at PCI level +- Only `/dev/dri` devices were exposed (insufficient for Windows) + +## Solution Overview + +1. Enable AMD IOMMU in kernel boot parameters +2. Bind GPU and audio devices to `vfio-pci` driver +3. Configure QEMU to pass through PCI devices +4. Restart container and install AMD drivers in Windows + +## Hardware Details + +- **GPU**: AMD Strix Halo iGPU (PCI: c5:00.0, ID: 1002:1586) +- **Audio**: HDMI Audio Controller (PCI: c5:00.1, ID: 1002:1640) + +## Setup Instructions + +### Step 1: Run Setup Script + +```bash +cd /mnt/shared/DEV/repos/d-popov.com/scripts/portainer-compose-stacks/windows +sudo ./setup-gpu-passthrough.sh +``` + +This script will: +- Enable IOMMU in GRUB (`amd_iommu=on iommu=pt`) +- Configure VFIO to claim the GPU devices +- Update initramfs with VFIO modules +- Create necessary configuration files + +### Step 2: Reboot + +```bash +sudo reboot +``` + +**IMPORTANT**: The system MUST be rebooted for IOMMU and VFIO changes to take effect. + +### Step 3: Verify Setup (After Reboot) + +```bash +cd /mnt/shared/DEV/repos/d-popov.com/scripts/portainer-compose-stacks/windows +./verify-gpu-passthrough.sh +``` + +Expected output: +- ✓ IOMMU is enabled +- ✓ vfio_pci module loaded +- ✓ GPU bound to vfio-pci +- ✓ Audio bound to vfio-pci +- ✓ /dev/vfio/vfio exists + +### Step 4: Start Windows Container + +```bash +cd /mnt/shared/DEV/repos/d-popov.com/scripts/portainer-compose-stacks/windows +docker-compose down # Stop if running +docker-compose up -d +``` + +### Step 5: Install AMD Drivers in Windows + +1. Connect to Windows via RDP: `localhost:3389` +2. Open Device Manager +3. You should see "AMD Display Adapter" (may show with warning) +4. Download AMD Radeon drivers for Windows 11 +5. Install the drivers +6. Reboot Windows +7. Verify GPU in Task Manager → Performance → GPU + +## Docker Compose Configuration + +The configuration includes: + +```yaml +environment: + GPU: "Y" + ARGUMENTS: "-device vfio-pci,host=c5:00.0,addr=0x02,multifunction=on -device vfio-pci,host=c5:00.1,addr=0x02.1" +devices: + - /dev/vfio/vfio +``` + +This passes through: +- **c5:00.0**: AMD GPU +- **c5:00.1**: HDMI Audio +- **addr=0x02**: Virtual PCI slot in guest +- **multifunction=on**: Allows multiple functions on same slot + +## Troubleshooting + +### GPU still shows VirtIO after reboot + +Check if GPU is bound to vfio-pci: +```bash +lspci -nnk -s c5:00.0 +``` + +Should show: `Kernel driver in use: vfio-pci` + +If not, manually bind: +```bash +echo "0000:c5:00.0" | sudo tee /sys/bus/pci/drivers/amdgpu/unbind +echo "1002 1586" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id +``` + +### IOMMU not enabled after reboot + +Check kernel parameters: +```bash +cat /proc/cmdline | grep iommu +``` + +Should show: `amd_iommu=on iommu=pt` + +If not present: +```bash +sudo nano /etc/default/grub +# Add to GRUB_CMDLINE_LINUX_DEFAULT: amd_iommu=on iommu=pt +sudo update-grub +sudo reboot +``` + +### Container fails to start + +Check logs: +```bash +docker logs windows +``` + +Common issues: +- VFIO device not found: Run verification script +- Permission denied: Ensure container is `privileged: true` +- Device busy: Another driver may be using the GPU + +### GPU not detected in Windows + +1. Check Device Manager for unknown devices +2. Try installing AMD Chipset drivers first +3. Use AMD Auto-Detect tool for driver installation +4. Check Windows Event Viewer for driver errors + +## Performance Notes + +- **Full GPU acceleration**: Direct PCI passthrough provides near-native performance +- **VRAM**: Windows will see the full iGPU VRAM allocation +- **Display output**: Use RDP/VNC (no physical display from container) +- **3D acceleration**: Fully supported (DirectX, OpenGL, Vulkan) + +## Limitations + +- Host cannot use the iGPU while passed through to the container +- Physical display outputs from the GPU won't work (headless only) +- USB devices may need separate passthrough configuration +- Some GPU features may require CPU pinning for best performance + +## Files Modified + +- `/etc/default/grub` - IOMMU kernel parameters +- `/etc/modprobe.d/vfio.conf` - VFIO driver binding +- `/etc/initramfs-tools/modules` - VFIO modules in initramfs +- `docker-compose.yml` - PCI passthrough arguments + +## Reverting Changes + +To restore GPU to host: + +1. Remove VFIO configuration: +```bash +sudo rm /etc/modprobe.d/vfio.conf +``` + +2. Edit GRUB to remove `vfio-pci.ids=...`: +```bash +sudo nano /etc/default/grub +sudo update-grub +``` + +3. Update initramfs and reboot: +```bash +sudo update-initramfs -u +sudo reboot +``` + +## References + +- [dockurr/windows](https://github.com/dockur/windows) - Windows container image +- [AMD GPU Passthrough Guide](https://wiki.archlinux.org/title/PCI_passthrough_via_OVMF) +- [VFIO Documentation](https://www.kernel.org/doc/html/latest/driver-api/vfio.html) + + + + + + + diff --git a/portainer-compose-stacks/windows/docker-compose.yml b/portainer-compose-stacks/windows/docker-compose.yml index af452d6..906e117 100644 --- a/portainer-compose-stacks/windows/docker-compose.yml +++ b/portainer-compose-stacks/windows/docker-compose.yml @@ -1,20 +1,25 @@ services: windows: image: dockurr/windows # https://github.com/dockur/windows - container_name: windows + container_name: windows2 environment: VERSION: "11" RAM_SIZE: "8G" CPU_CORES: "4" + GPU: "Y" + ARGUMENTS: "-device vfio-pci,host=c5:00.0,addr=0x05,multifunction=on -device vfio-pci,host=c5:00.1,addr=0x05.1" devices: - /dev/kvm - /dev/net/tun - /dev/nvme0n1p8:/disk2 + - /dev/vfio/vfio + - /dev/vfio/20 + - /dev/vfio/21 cap_add: - NET_ADMIN privileged: true ports: - #- 445:445 + - 445:445 - 1433:1433 - 8006:8006 - 3389:3389/tcp diff --git a/portainer-compose-stacks/windows/fix-grub-iommu.sh b/portainer-compose-stacks/windows/fix-grub-iommu.sh new file mode 100644 index 0000000..cbca57b --- /dev/null +++ b/portainer-compose-stacks/windows/fix-grub-iommu.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Fix GRUB IOMMU conflict - remove amd_iommu=off that's blocking the passthrough + +set -e + +if [ "$EUID" -ne 0 ]; then + echo "Please run as root: sudo $0" + exit 1 +fi + +echo "=== Fixing GRUB IOMMU Configuration ===" +echo "" + +GRUB_FILE="/etc/default/grub" + +# Backup +cp "$GRUB_FILE" "$GRUB_FILE.backup.$(date +%Y%m%d-%H%M%S)" +echo "✓ Backup created" + +# Remove the conflicting amd_iommu=off +sed -i 's/ amd_iommu=off//' "$GRUB_FILE" + +echo "✓ Removed amd_iommu=off from GRUB config" +echo "" +echo "New GRUB_CMDLINE_LINUX_DEFAULT:" +grep "GRUB_CMDLINE_LINUX_DEFAULT" "$GRUB_FILE" +echo "" + +# Update GRUB +echo "Updating GRUB..." +update-grub + +echo "" +echo "=== Fix Complete ===" +echo "" +echo "⚠️ REBOOT NOW to apply changes: sudo reboot" +echo "" +echo "After reboot, verify with:" +echo " ./verify-gpu-passthrough.sh" + + + diff --git a/portainer-compose-stacks/windows/fix-grub-manual.sh b/portainer-compose-stacks/windows/fix-grub-manual.sh new file mode 100644 index 0000000..3930fd9 --- /dev/null +++ b/portainer-compose-stacks/windows/fix-grub-manual.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Manual GRUB fix - completely rewrite the GRUB_CMDLINE_LINUX_DEFAULT line + +set -e + +if [ "$EUID" -ne 0 ]; then + echo "Please run as root: sudo $0" + exit 1 +fi + +echo "=== Fixing GRUB Configuration (Manual Method) ===" +echo "" + +GRUB_FILE="/etc/default/grub" + +# Backup +cp "$GRUB_FILE" "$GRUB_FILE.backup.manual.$(date +%Y%m%d-%H%M%S)" +echo "✓ Backup created" + +# Replace the entire line with a clean version +sed -i 's|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT="quiet splash amd_iommu=on iommu=pt vfio-pci.ids=1002:1586,1002:1640 amdgpu.gttsize=131072 ttm.pages_limit=33554432 amdgpu.si_support=1 amdgpu.cik_support=1 radeon.si_support=0 radeon.cik_support=0"|' "$GRUB_FILE" + +echo "✓ GRUB configuration updated" +echo "" +echo "New configuration:" +grep "GRUB_CMDLINE_LINUX_DEFAULT" "$GRUB_FILE" +echo "" + +# Update GRUB +echo "Updating GRUB..." +update-grub + +echo "" +echo "=== Fix Complete ===" +echo "" +echo "Changes made:" +echo " - Removed: amd_iommu=off" +echo " - Added: amd_iommu=on iommu=pt" +echo " - Added: vfio-pci.ids=1002:1586,1002:1640" +echo "" +echo "⚠️ REBOOT REQUIRED: sudo reboot" +echo "" +echo "After reboot, check:" +echo " cat /proc/cmdline | grep amd_iommu" +echo " Should show ONLY: amd_iommu=on" + + diff --git a/portainer-compose-stacks/windows/setup-gpu-passthrough.sh b/portainer-compose-stacks/windows/setup-gpu-passthrough.sh new file mode 100644 index 0000000..024f0ab --- /dev/null +++ b/portainer-compose-stacks/windows/setup-gpu-passthrough.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# GPU Passthrough Setup for AMD Strix Halo iGPU to Windows Container +# This script enables IOMMU and configures VFIO-PCI for GPU passthrough + +set -e + +echo "=== AMD Strix Halo GPU Passthrough Setup ===" +echo "" + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo "Please run as root (sudo)" + exit 1 +fi + +# GPU PCI IDs +GPU_PCI="0000:c5:00.0" +GPU_AUDIO_PCI="0000:c5:00.1" +GPU_VENDOR_ID="1002" +GPU_DEVICE_ID="1586" + +# Check current IOMMU status +echo "Current IOMMU status:" +cat /proc/cmdline | grep -o "amd_iommu=[^ ]*" || echo "IOMMU not configured in kernel parameters" +echo "" + +# Step 1: Enable IOMMU in GRUB +echo "Step 1: Checking GRUB configuration..." +GRUB_FILE="/etc/default/grub" + +if ! grep -q "amd_iommu=on" "$GRUB_FILE"; then + echo "Adding IOMMU parameters to GRUB..." + cp "$GRUB_FILE" "$GRUB_FILE.backup.$(date +%Y%m%d-%H%M%S)" + + # Update GRUB_CMDLINE_LINUX_DEFAULT + sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="\(.*\)"/GRUB_CMDLINE_LINUX_DEFAULT="\1 amd_iommu=on iommu=pt vfio-pci.ids=1002:1586,1002:1640"/' "$GRUB_FILE" + + # Clean up double spaces + sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT=" /GRUB_CMDLINE_LINUX_DEFAULT="/' "$GRUB_FILE" + + echo "Updating GRUB..." + update-grub + + echo "" + echo "✓ GRUB updated. IOMMU will be enabled after reboot." +else + echo "✓ IOMMU already configured in GRUB" +fi + +# Step 2: Configure VFIO modules +echo "" +echo "Step 2: Configuring VFIO modules..." +VFIO_CONF="/etc/modprobe.d/vfio.conf" + +cat > "$VFIO_CONF" << EOF +# Bind AMD Strix Halo GPU to VFIO for passthrough +options vfio-pci ids=1002:1586,1002:1640 +softdep amdgpu pre: vfio-pci +softdep snd_hda_intel pre: vfio-pci +EOF + +echo "✓ VFIO configuration created" + +# Step 3: Update initramfs +echo "" +echo "Step 3: Adding VFIO modules to initramfs..." +MODULES_FILE="/etc/initramfs-tools/modules" + +if ! grep -q "vfio_pci" "$MODULES_FILE"; then + cat >> "$MODULES_FILE" << EOF + +# VFIO modules for GPU passthrough +vfio +vfio_iommu_type1 +vfio_pci +vfio_virqfd +EOF + echo "✓ VFIO modules added to initramfs" +else + echo "✓ VFIO modules already in initramfs" +fi + +echo "" +echo "Updating initramfs..." +update-initramfs -u + +# Step 4: Summary and next steps +echo "" +echo "=== Setup Complete ===" +echo "" +echo "Configuration applied:" +echo " - IOMMU enabled in GRUB (amd_iommu=on iommu=pt)" +echo " - GPU (1002:1586) bound to vfio-pci driver" +echo " - Audio (1002:1640) bound to vfio-pci driver" +echo " - Initramfs updated with VFIO modules" +echo "" +echo "⚠️ REBOOT REQUIRED to apply changes by running: 'sudo update-grub' and 'sudo reboot'" +echo "" +echo "After reboot:" +echo " 1. Verify IOMMU is enabled: dmesg | grep -i iommu" +echo " 2. Check GPU binding: lspci -nnk -d 1002:1586" +echo " 3. Start Windows container: docker-compose up -d" +echo " 4. Install AMD drivers in Windows" +echo "" +echo "To reboot now, run: reboot" + + + diff --git a/portainer-compose-stacks/windows/verify-gpu-passthrough.sh b/portainer-compose-stacks/windows/verify-gpu-passthrough.sh new file mode 100644 index 0000000..6b75507 --- /dev/null +++ b/portainer-compose-stacks/windows/verify-gpu-passthrough.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Verification script for GPU Passthrough Setup +# Run this after reboot to verify IOMMU and VFIO configuration + +echo "=== GPU Passthrough Verification ===" +echo "" + +# Check 1: IOMMU enabled +echo "1. Checking IOMMU status..." +if dmesg | grep -qi "AMD-Vi: AMD IOMMUv2 loaded"; then + echo " ✓ IOMMU is enabled" +else + echo " ✗ IOMMU not detected in dmesg" + echo " Boot parameters: $(cat /proc/cmdline | grep -o 'amd_iommu=[^ ]*\|iommu=[^ ]*')" +fi +echo "" + +# Check 2: VFIO module loaded +echo "2. Checking VFIO modules..." +if lsmod | grep -q vfio_pci; then + echo " ✓ vfio_pci module loaded" +else + echo " ✗ vfio_pci module not loaded" + echo " Try: modprobe vfio-pci" +fi +echo "" + +# Check 3: GPU bound to VFIO +echo "3. Checking GPU (c5:00.0) driver binding..." +GPU_DRIVER=$(lspci -nnk -s c5:00.0 | grep "Kernel driver in use" | awk '{print $5}') +if [ "$GPU_DRIVER" = "vfio-pci" ]; then + echo " ✓ GPU bound to vfio-pci" +else + echo " ✗ GPU bound to: ${GPU_DRIVER:-none}" + echo " Expected: vfio-pci" +fi + +lspci -nnk -s c5:00.0 | head -5 +echo "" + +# Check 4: Audio bound to VFIO +echo "4. Checking GPU Audio (c5:00.1) driver binding..." +AUDIO_DRIVER=$(lspci -nnk -s c5:00.1 | grep "Kernel driver in use" | awk '{print $5}') +if [ "$AUDIO_DRIVER" = "vfio-pci" ]; then + echo " ✓ Audio bound to vfio-pci" +else + echo " ✗ Audio bound to: ${AUDIO_DRIVER:-none}" + echo " Expected: vfio-pci" +fi + +lspci -nnk -s c5:00.1 | head -5 +echo "" + +# Check 5: IOMMU groups +echo "5. Checking IOMMU groups..." +if [ -e /sys/bus/pci/devices/0000:c5:00.0/iommu_group ]; then + GPU_GROUP=$(basename $(readlink /sys/bus/pci/devices/0000:c5:00.0/iommu_group)) + echo " ✓ GPU IOMMU group: $GPU_GROUP" + echo " Devices in group:" + ls -1 /sys/bus/pci/devices/0000:c5:00.0/iommu_group/devices/ | sed 's/^/ - /' +else + echo " ✗ IOMMU group not found for GPU" +fi +echo "" + +# Check 6: VFIO devices +echo "6. Checking /dev/vfio devices..." +if [ -e /dev/vfio/vfio ]; then + echo " ✓ /dev/vfio/vfio exists" + ls -la /dev/vfio/ | sed 's/^/ /' +else + echo " ✗ /dev/vfio/vfio not found" +fi +echo "" + +# Summary +echo "=== Summary ===" +if [ "$GPU_DRIVER" = "vfio-pci" ] && [ "$AUDIO_DRIVER" = "vfio-pci" ] && [ -e /dev/vfio/vfio ]; then + echo "✓ GPU passthrough is configured correctly!" + echo "" + echo "Next steps:" + echo " 1. Start the Windows container: cd /mnt/shared/DEV/repos/d-popov.com/scripts/portainer-compose-stacks/windows && docker-compose up -d" + echo " 2. Connect to Windows via RDP: localhost:3389" + echo " 3. Install AMD Radeon drivers in Windows" + echo " 4. GPU should appear as 'AMD Strix Halo' in Device Manager" +else + echo "✗ GPU passthrough configuration needs attention" + echo "" + echo "If GPU is not bound to vfio-pci:" + echo " - Check /etc/modprobe.d/vfio.conf" + echo " - Run: sudo update-initramfs -u" + echo " - Reboot again" +fi + +