Kimi K2.5 Download: Complete Local Setup and Installation Guide

Kimi K2.5 Local Storage Requirements (planning reference):
┌─────────────────────────────────────────────────────┐
│ Official checkpoint files:     ~2,000 GB            │
│ Runtime cache/temp files:     100-300 GB            │
│ Logs and deployment buffer:   100-300 GB            │
├─────────────────────────────────────────────────────┤
│ Full-precision total:       ~2,200+ GB              │
│ Quantized/community setups:    600+ GB              │
└─────────────────────────────────────────────────────┘

# GPU memory calculation for Kimi K2.5
class GPUMemoryCalculator:
    def __init__(self):
        self.model_params = 1e12  # 1 trillion
        self.bytes_per_param = 2   # FP16
        self.activation_factor = 4  # Activation overhead
    
    def calculate_required_memory(self, batch_size=1, seq_length=128000):
        # Model weights
        model_memory = self.model_params * self.bytes_per_param / (1024**3)  # GB
        
        # Activations for sequence
        activation_memory = (
            batch_size * seq_length * self.activation_factor / (1024**3)
        )
        
        # KV cache for 128K context
        kv_cache_per_layer = 128 * seq_length * 2 / (1024**3)  # GB
        total_kv_cache = kv_cache_per_layer * 96  # 96 layers
        
        total = model_memory + activation_memory + total_kv_cache
        
        return {
            "model_weights_gb": model_memory,
            "activations_gb": activation_memory,
            "kv_cache_gb": total_kv_cache,
            "total_gb": total,
            "recommended_gpus": self._recommend_gpus(total)
        }
    
    def _recommend_gpus(self, total_memory_gb):
        a100_80gb = 80
        num_gpus = (total_memory_gb / a100_80gb) * 1.2  # 20% margin
        return max(2, int(num_gpus))

# Install Hugging Face CLI
pip install huggingface-cli

# Login (requires authentication)
huggingface-cli login

# Download Kimi K2.5 weights
# Note: Requires accepting license terms on Hugging Face
huggingface-cli download moonshotai/Kimi-K2.5 \
  --local-dir ./kimi-k2-5 \
  --local-dir-use-symlinks False

# Install Git LFS
git lfs install

# Clone the repository
git clone https://huggingface.co/moonshotai/Kimi-K2.5
cd Kimi-K2.5

# Pull LFS files (large model weights)
git lfs pull

# Python script for downloading model shards
import requests
from tqdm import tqdm
import os

def download_kimi_weights(output_dir="./kimi-k2-5"):
    """Download Kimi K2.5 model weights"""
    
    base_url = "https://huggingface.co/moonshotai/Kimi-K2.5/resolve/main"
    
    files = [
        "config.json",
        "tokenizer.json",
        "model.safetensors.index.json",
        # Shards will be listed in index.json
    ]
    
    os.makedirs(output_dir, exist_ok=True)
    
    for file in files:
        url = f"{base_url}/{file}"
        response = requests.get(url, stream=True)
        
        total_size = int(response.headers.get('content-length', 0))
        
        with open(os.path.join(output_dir, file), 'wb') as f:
            with tqdm(total=total_size, unit='B', unit_scale=True, desc=file) as pbar:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
                        pbar.update(len(chunk))
    
    print(f"Downloaded to {output_dir}")
    print("Note: Official full-precision checkpoints are roughly in the 2TB class")

# Install Ollama
curl -fsSL https://ollama.com/install.sh | sh

# Pull Kimi K2.5 cloud entry
ollama pull kimi-k2.5:cloud

# Run via Ollama
ollama run kimi-k2.5:cloud

# Test the model
>>> What is the capital of France?
The capital of France is Paris.

# Note: This Ollama entry uses cloud inference and does not download full local weights.

# Install vLLM
pip install vllm

# Download and run Kimi K2.5
python -m vllm.entrypoints.openai.api_server \
  --model moonshotai/Kimi-K2.5 \
  --tensor-parallel-size 4 \
  --pipeline-parallel-size 2 \
  --max-model-len 128000 \
  --gpu-memory-utilization 0.95 \
  --port 8000

# Dockerfile for Kimi K2.5
FROM nvidia/cuda:12.1-devel-ubuntu22.04

WORKDIR /app

# Install dependencies
RUN apt-get update && apt-get install -y \
    python3-pip \
    git \
    git-lfs \
    && rm -rf /var/lib/apt/lists/*

# Install Python packages
RUN pip install torch transformers accelerate vllm

# Download model (or mount as volume)
RUN git lfs install && \
    git clone https://huggingface.co/moonshotai/Kimi-K2.5 /models/kimi-k2-5

# Expose API port
EXPOSE 8000

# Start the server
CMD python -m vllm.entrypoints.openai.api_server \
    --model /models/kimi-k2-5 \
    --tensor-parallel-size 4 \
    --max-model-len 128000 \
    --host 0.0.0.0 \
    --port 8000

# Build and run
docker build -t kimi-k2-5 .
docker run --gpus all -p 8000:8000 -v /path/to/models:/models kimi-k2-5

# Direct inference with Transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(
    "moonshotai/Kimi-K2.5",
    trust_remote_code=True
)

# Load model (requires significant GPU memory)
model = AutoModelForCausalLM.from_pretrained(
    "moonshotai/Kimi-K2.5",
    torch_dtype=torch.float16,
    device_map="auto",  # Automatically distribute across GPUs
    trust_remote_code=True
)

# Generate text
inputs = tokenizer("Hello, how are you?", return_tensors="pt")
outputs = model.generate(
    **inputs,
    max_new_tokens=100,
    temperature=0.7
)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(response)

# FastAPI server for Kimi K2.5
from fastapi import FastAPI
from pydantic import BaseModel
from typing import List, Optional
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

app = FastAPI(title="Kimi K2.5 Local API")

# Global model and tokenizer
model = None
tokenizer = None

class ChatMessage(BaseModel):
    role: str
    content: str

class ChatRequest(BaseModel):
    model: str
    messages: List[ChatMessage]
    temperature: Optional[float] = 0.7
    max_tokens: Optional[int] = 1024

@app.on_event("startup")
async def load_model():
    global model, tokenizer
    
    print("Loading Kimi K2.5... This may take several minutes.")
    
    tokenizer = AutoTokenizer.from_pretrained(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        "moonshotai/Kimi-K2.5",
        torch_dtype=torch.float16,
        device_map="auto",
        trust_remote_code=True
    )
    
    print("Model loaded successfully!")

@app.post("/v1/chat/completions")
async def chat_completion(request: ChatRequest):
    # Format messages
    prompt = tokenizer.apply_chat_template(
        request.messages,
        tokenize=False,
        add_generation_prompt=True
    )
    
    # Tokenize
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    
    # Generate
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=request.max_tokens,
            temperature=request.temperature,
            do_sample=True
        )
    
    # Decode
    response_text = tokenizer.decode(
        outputs[0][inputs.input_ids.shape[1]:],
        skip_special_tokens=True
    )
    
    return {
        "id": "chatcmpl-local",
        "object": "chat.completion",
        "model": request.model,
        "choices": [{
            "index": 0,
            "message": {
                "role": "assistant",
                "content": response_text
            }
        }]
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

# GGUF builds are community-maintained (Moonshot's official repo ships original checkpoints)
COMMUNITY_REPO="<community-org>/Kimi-K2.5-GGUF"
GGUF_FILE="kimi-k2-5-Q4_K_M.gguf"
huggingface-cli download ${COMMUNITY_REPO} ${GGUF_FILE} --local-dir ./models

# Run with llama.cpp
./main -m ./models/kimi-k2-5-Q4_K_M.gguf \
  -c 32768 \
  -n 512 \
  -p "Hello, my name is"

# Using AWQ for 4-bit quantization
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "moonshotai/Kimi-K2.5"
quant_path = "kimi-k2-5-awq"
quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4}

# Load model
model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

# Quantize
model.quantize(tokenizer, quant_config=quant_config)

# Save
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)

# kimi-k2-5-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kimi-k2-5
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kimi-k2-5
  template:
    metadata:
      labels:
        app: kimi-k2-5
    spec:
      nodeSelector:
        node-type: gpu-a100
      containers:
      - name: kimi-k2-5
        image: your-registry/kimi-k2-5:latest
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: 4
          requests:
            nvidia.com/gpu: 4
        volumeMounts:
        - name: model-storage
          mountPath: /models
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: kimi-model-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: kimi-k2-5-service
spec:
  selector:
    app: kimi-k2-5
  ports:
  - port: 8000
    targetPort: 8000
  type: LoadBalancer

Under the assumptions above, local deployment breaks even at:
- ~46M tokens/month (input + output weighted average)
- ~18 months at 100M tokens/month
- ~36 months at 50M tokens/month

# Solutions for OOM errors

# 1. Reduce batch size
model.generate(**inputs, batch_size=1)

# 2. Enable gradient checkpointing (for training)
model.gradient_checkpointing_enable()

# 3. Use CPU offloading
from accelerate import load_checkpoint_and_dispatch

model = load_checkpoint_and_dispatch(
    model,
    checkpoint="moonshotai/Kimi-K2.5",
    device_map="auto",
    offload_folder="offload"
)

# 4. Reduce context length
max_length = 65536  # Instead of 128000

# Optimization techniques

# 1. Use Flash Attention
from transformers import AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    "moonshotai/Kimi-K2.5",
    attn_implementation="flash_attention_2",
    torch_dtype=torch.float16
)

# 2. Compile model (PyTorch 2.0+)
model = torch.compile(model)

# 3. Use speculative decoding
from transformers import SpeculativeDecoding

speculator = SpeculativeDecoding(
    draft_model="small-draft-model",
    target_model=model
)

# API key authentication
from fastapi import Security, HTTPException
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials

security = HTTPBearer()

API_KEYS = {"your-secure-api-key-here": "admin"}

async def verify_token(credentials: HTTPAuthorizationCredentials = Security(security)):
    token = credentials.credentials
    if token not in API_KEYS:
        raise HTTPException(status_code=401, detail="Invalid API key")
    return API_KEYS[token]

@app.post("/v1/chat/completions")
async def chat_completion(
    request: ChatRequest,
    user: str = Security(verify_token)
):
    # Process request
    pass