LLM Context Caching: KV Cache Architecture

import anthropic

client = anthropic.Anthropic()

response = client.messages.create(
    model="claude-3-7-sonnet-20250219",
    max_tokens=1024,
    system=[
        {
            "type": "text",
            "text": "You are a senior codebase architect with access to our entire internal API docs...",
            # Pinned system prompt cache breakpoint
            "cache_control": {"type": "ephemeral"}
        }
    ],
    tools=[
        {
            "name": "large_mcp_toolkit",
            "description": "Enterprise tool suite",
            "input_schema": {...},
            # Tool definition cache breakpoint
            "cache_control": {"type": "ephemeral"}
        }
    ],
    messages=[
        {"role": "user", "content": "Analyze performance bottleneck in auth/middleware.go"}
    ]
)

# Inspect Cache Performance
usage = response.usage
print(f"Cached tokens read: {usage.cache_read_input_tokens} (Billed at 10% rate!)")
print(f"New tokens written: {usage.cache_creation_input_tokens}")

from google import genai
from google.genai import types

client = genai.Client()

# 1. Create durable cache for 100,000 tokens of documentation
cache = client.cached_contents.create(
    model="gemini-2.0-flash-exp",
    config=types.CreateCachedContentConfig(
        contents=[large_codebase_documents],
        system_instruction="You are an enterprise code assistant.",
        ttl="3600s" # Cache remains active for 1 hour
    )
)

# 2. Query using the cached content reference
response = client.models.generate_content(
    model="gemini-2.0-flash-exp",
    contents="Where is the rate limiter implemented?",
    config=types.GenerateContentConfig(
        cached_content=cache.name
    )
)
print("Gemini response retrieved with 75% cached discount!")