83 lines
2.3 KiB
Bash
83 lines
2.3 KiB
Bash
#!/bin/bash
|
|
|
|
# Manual Docker AI Model Setup
|
|
# This creates a Docker-based AI model runner similar to Docker Model Runner
|
|
|
|
echo "=== Manual Docker AI Model Setup ==="
|
|
echo ""
|
|
|
|
# Create a directory for AI models
|
|
mkdir -p ~/docker-ai-models
|
|
cd ~/docker-ai-models
|
|
|
|
# Create Docker Compose file for AI models
|
|
cat > docker-compose.yml << 'EOF'
|
|
version: '3.8'
|
|
|
|
services:
|
|
llama-cpp-server:
|
|
image: ghcr.io/ggerganov/llama.cpp:server
|
|
ports:
|
|
- "8080:8080"
|
|
volumes:
|
|
- ./models:/models
|
|
environment:
|
|
- MODEL_PATH=/models
|
|
command: --model /models/llama-2-7b-chat.Q4_K_M.gguf --host 0.0.0.0 --port 8080
|
|
|
|
text-generation-webui:
|
|
image: ghcr.io/oobabooga/text-generation-webui:latest
|
|
ports:
|
|
- "7860:7860"
|
|
volumes:
|
|
- ./models:/models
|
|
environment:
|
|
- CLI_ARGS=--listen --listen-port 7860 --model-dir /models
|
|
command: python server.py --listen --listen-port 7860 --model-dir /models
|
|
EOF
|
|
|
|
echo "Docker Compose file created!"
|
|
|
|
# Create a model download script
|
|
cat > download_models.sh << 'EOF'
|
|
#!/bin/bash
|
|
|
|
echo "=== Downloading AI Models ==="
|
|
echo ""
|
|
|
|
# Create models directory
|
|
mkdir -p models
|
|
|
|
# Download Llama 2 7B Chat (GGUF format)
|
|
echo "Downloading Llama 2 7B Chat..."
|
|
wget -O models/llama-2-7b-chat.Q4_K_M.gguf \
|
|
"https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf"
|
|
|
|
# Download Mistral 7B (GGUF format)
|
|
echo "Downloading Mistral 7B..."
|
|
wget -O models/mistral-7b-instruct-v0.1.Q4_K_M.gguf \
|
|
"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q4_K_M.gguf"
|
|
|
|
echo "Models downloaded successfully!"
|
|
echo "You can now run: docker-compose up"
|
|
EOF
|
|
|
|
chmod +x download_models.sh
|
|
|
|
echo ""
|
|
echo "=== Setup Complete! ==="
|
|
echo ""
|
|
echo "To get started:"
|
|
echo "1. Run: ./download_models.sh # Download models"
|
|
echo "2. Run: docker-compose up # Start AI services"
|
|
echo ""
|
|
echo "=== Available Services ==="
|
|
echo "- Llama.cpp Server: http://localhost:8080"
|
|
echo "- Text Generation WebUI: http://localhost:7860"
|
|
echo ""
|
|
echo "=== API Usage ==="
|
|
echo "You can interact with the models via HTTP API:"
|
|
echo "curl -X POST http://localhost:8080/completion \\"
|
|
echo " -H 'Content-Type: application/json' \\"
|
|
echo " -d '{\"prompt\": \"Hello, how are you?\", \"n_predict\": 100}'"
|