TLS, Certificates & Truststores:
The Production Security Guide

# 1. Inspect existing certificates in Java cacerts
keytool -list -v \
  -keystore $JAVA_HOME/lib/security/cacerts \
  -storepass changeit | grep -i "Alias name:"

# 2. Import custom corporate Root CA into cacerts
keytool -importcert \
  -alias "corp-internal-root-ca" \
  -file internal-ca.crt \
  -keystore $JAVA_HOME/lib/security/cacerts \
  -storepass changeit \
  -noprompt

# 3. Specify custom Keystore and Truststore at application startup
java -Djavax.net.ssl.trustStore=/etc/ssl/my-truststore.p12 \
     -Djavax.net.ssl.trustStorePassword=truststoreSecret \
     -Djavax.net.ssl.trustStoreType=PKCS12 \
     -Djavax.net.ssl.keyStore=/etc/ssl/my-keystore.p12 \
     -Djavax.net.ssl.keyStorePassword=keystoreSecret \
     -Djavax.net.ssl.keyStoreType=PKCS12 \
     -jar my-service.jar

# Debian / Ubuntu Systems:
# Copy certificate (.crt) into ca-certificates folder
sudo cp internal-ca.crt /usr/local/share/ca-certificates/internal-ca.crt
sudo update-ca-certificates
# Bundle is assembled at: /etc/ssl/certs/ca-certificates.crt

# RHEL / CentOS / Rocky Linux / Fedora:
# Copy certificate into the ca-trust anchors directory
sudo cp internal-ca.crt /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract
# Bundle is assembled at: /etc/pki/tls/certs/ca-bundle.crt

# Test system trust via curl:
curl -v https://internal-service.corp.local

# Python: Force requests / urllib3 / httpx to use custom CA
export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

# In Python code directly:
import httpx
client = httpx.Client(verify="/path/to/internal-ca.crt")

# Node.js: Point to extra CA bundle (avoids ignoring custom CAs)
export NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/internal-ca.crt

# Go: Loads system pool by default via crypto/tls
pool, _ := x509.SystemCertPool()
caCert, _ := os.ReadFile("internal-ca.crt")
pool.AppendCertsFromPEM(caCert)
tlsConfig := &tls.Config{RootCAs: pool}

FROM python:3.11-slim

# 1. Copy enterprise root certificate into container
COPY ./corp-root-ca.crt /usr/local/share/ca-certificates/corp-root-ca.crt

# 2. Update Debian system truststore & configure pip / node envs
RUN apt-get update && apt-get install -y ca-certificates && \
    update-ca-certificates && \
    rm -rf /var/lib/apt/lists/*

# 3. Inform Python, cURL, and Node of the unified bundle
ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
    NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt \
    PIP_CERT=/etc/ssl/certs/ca-certificates.crt

# Now pip install works behind corporate SSL-inspecting proxies!
RUN pip install --no-cache-dir fastapi uvicorn httpx

import httpx

# The agent loads its own identity (Keystore) AND trusted CA (Truststore)
agent_client_cert = ("certs/agent-client.crt", "certs/agent-client.key")
trusted_ca_bundle = "certs/mesh-ca-chain.crt"

# Establish zero-trust mTLS connection
with httpx.Client(cert=agent_client_cert, verify=trusted_ca_bundle) as client:
    response = client.post(
        "https://vllm-cluster.internal:8443/v1/chat/completions",
        json={
            "model": "deepseek-r1:70b",
            "messages": [{"role": "user", "content": "Execute agent task"}]
        }
    )
    print(f"Status: {response.status_code}")
    print(f"Response: {response.json()}")

server {
    listen 8443 ssl http2;
    server_name vllm-cluster.internal;

    # 1. Server Keystore (Server's own identity)
    ssl_certificate     /etc/ssl/server/server.crt;
    ssl_certificate_key /etc/ssl/server/server.key;

    # 2. Server Truststore for verifying incoming Clients
    ssl_client_certificate /etc/ssl/ca/agent-mesh-ca.crt;
    ssl_verify_client       on; # Rejects any client without a valid CA-signed cert!
    ssl_verify_depth        2;

    # Pass authenticated client identity to upstream application
    location / {
        proxy_pass http://localhost:8000;
        proxy_set_header X-Client-DN $ssl_client_s_dn;
        proxy_set_header X-Client-Verify $ssl_client_verify;
    }
}

# 1. ClusterIssuer for Let's Encrypt Production
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: devops@aiagent.org
    privateKeySecretRef:
      name: letsencrypt-prod-account-key
    solvers:
    - http01:
        ingress:
          class: nginx

---
# 2. Ingress with Automatic TLS Generation & 30-Day Renewal
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: agent-api-ingress
  annotations:
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - api.aiagent.org
    secretName: api-aiagent-org-tls
  rules:
  - host: api.aiagent.org
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: agent-backend
            port:
              number: 8080

groups:
- name: tls_alerts
  rules:
  # Alert if certificate will expire within 14 days
  - alert: TLSCertificateExpiringSoon
    expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 14
    for: 1h
    labels:
      severity: warning
    annotations:
      summary: "TLS certificate for {{ $labels.instance }} expiring soon"
      description: "Certificate expires in {{ $value | humanizeDuration }}. Check automated renewal!"

  # Critical alert if certificate will expire within 48 hours
  - alert: TLSCertificateExpiryCritical
    expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 2
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "EMERGENCY: TLS certificate for {{ $labels.instance }} expires in under 48 hours"