Run Local LLMs on Commodity Hardware: The Complete Setup Guide

# 1. Clone & Compile with Hardware Acceleration
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

# For NVIDIA GPUs (CUDA):
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release -j $(nproc)

# For Apple Silicon (Metal):
cmake -B build -DGGML_METAL=ON
cmake --build build --config Release -j $(sysctl -n hw.ncpu)

# 2. Download GGUF Model from Hugging Face
wget -O Qwen2.5-14B-Instruct-Q4_K_M.gguf \
    https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF/resolve/main/qwen2.5-14b-instruct-q4_k_m.gguf

# 3. Launch High-Performance OpenAI-Compatible Server
./build/bin/llama-server \
    -m ./Qwen2.5-14B-Instruct-Q4_K_M.gguf \
    --host 0.0.0.0 \
    --port 8080 \
    -ngl 99 \             # Offload all 48 layers to GPU VRAM (Use e.g. 24 if VRAM is tight)
    -c 16384 \            # Context length: 16k tokens
    -b 512 \              # Batch size for prompt prefill
    -fa \                 # Enable FlashAttention (Saves ~40% KV cache VRAM & speeds prefill)
    -t 8                  # Number of CPU compute threads

version: '3.8'

services:
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    container_name: open-webui
    restart: unless-stopped
    ports:
      - "3000:8080"
    volumes:
      - open-webui-data:/app/backend/data
    environment:
      # Connects directly to host machine's Ollama daemon
      - OLLAMA_BASE_URL=http://host.docker.internal:11434
      - WEBUI_AUTH=true  # Multi-user login & private chat history
      - ENABLE_RAG_WEB_SEARCH=true # Optional web search integration
    extra_hosts:
      - "host.docker.internal:host-gateway"

volumes:
  open-webui-data:

from openai import OpenAI
import json

# 1. Initialize OpenAI client pointing to Local Ollama daemon
client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama", # Required dummy string; local server ignores auth
)

# 2. Define a local Python tool function
def get_system_load():
    """Simulated local machine tool execution."""
    return {"cpu_percent": 18.4, "vram_free_gb": 8.2, "status": "OPTIMAL"}

# 3. Query the local model with function calling schema
tools = [{
    "type": "function",
    "function": {
        "name": "get_system_load",
        "description": "Check current system CPU and VRAM hardware load",
        "parameters": {"type": "object", "properties": {}},
    }
}]

response = client.chat.completions.create(
    model="qwen2.5:14b", # Or llama3.1:8b
    messages=[
        {"role": "system", "content": "You are a local sysadmin agent."},
        {"role": "user", "content": "What is the current health of our machine?"}
    ],
    tools=tools,
    temperature=0.1
)

# Check if model triggered tool execution
message = response.choices[0].message
if message.tool_calls:
    tool_call = message.tool_calls[0]
    print(f"Agent decided to invoke local tool: {tool_call.function.name}")
    tool_output = get_system_load()
    print(f"Tool Output: {json.dumps(tool_output)}")
else:
    print(message.content)