Kubernetes v1.33 新特性深度解读：GA 特性全览与升级指南

resizePolicy:
  - resourceName: cpu
    restartPolicy: NotRequired   # CPU 调整无需重启
  - resourceName: memory
    restartPolicy: RestartContainer  # 内存调整需要重启容器

apiVersion: v1
kind: Pod
metadata:
  name: mysql-standalone
  namespace: production
spec:
  containers:
  - name: mysql
    image: mysql:8.0
    resources:
      requests:
        cpu: "2"
        memory: "4Gi"
      limits:
        cpu: "4"
        memory: "8Gi"
    resizePolicy:
    - resourceName: cpu
      restartPolicy: NotRequired
    - resourceName: memory
      restartPolicy: RestartContainer
    env:
    - name: MYSQL_ROOT_PASSWORD
      valueFrom:
        secretKeyRef:
          name: mysql-secret
          key: password

# 在线扩容 CPU，不重启 Pod
kubectl patch pod mysql-standalone -n production --type=json -p='[
  {
    "op": "replace",
    "path": "/spec/containers/0/resources/requests/cpu",
    "value": "3"
  },
  {
    "op": "replace",
    "path": "/spec/containers/0/resources/limits/cpu",
    "value": "6"
  }
]'

# 查看调整状态
kubectl get pod mysql-standalone -n production -o jsonpath='{.status.resize}'
# 输出: InProgress -> Deferred -> Infeasible -> "" (成功)

status:
  resize: ""          # 空字符串表示调整完成或无进行中的调整
  containerStatuses:
  - name: mysql
    allocatedResources:
      cpu: "3"        # 实际分配的资源（可能与 spec 不同步）
      memory: "4Gi"
    resources:
      requests:
        cpu: "3"
        memory: "4Gi"

apiVersion: v1
kind: Pod
metadata:
  name: app-with-logging
  namespace: production
spec:
  initContainers:
  # 原生 sidecar：在主容器之前启动，与主容器同生命周期
  - name: log-collector
    image: fluent/fluent-bit:3.0
    restartPolicy: Always          # 这是关键字段
    volumeMounts:
    - name: log-volume
      mountPath: /logs
    - name: fluent-bit-config
      mountPath: /fluent-bit/etc
    startupProbe:
      httpGet:
        path: /api/v1/health
        port: 2020
      initialDelaySeconds: 3
      periodSeconds: 5
      failureThreshold: 10
  containers:
  - name: app
    image: myapp:v2.0
    volumeMounts:
    - name: log-volume
      mountPath: /var/log/app
  volumes:
  - name: log-volume
    emptyDir: {}
  - name: fluent-bit-config
    configMap:
      name: fluent-bit-config

apiVersion: batch/v1
kind: Job
metadata:
  name: data-migration
spec:
  template:
    spec:
      initContainers:
      - name: istio-proxy
        image: istio/proxyv2:1.21.0
        restartPolicy: Always
        args: ["proxy", "sidecar"]
        env:
        - name: ISTIO_META_WORKLOAD_NAME
          value: data-migration
      containers:
      - name: migrator
        image: myapp/migrator:v1.0
        command: ["./migrate.sh"]
      restartPolicy: Never

apiVersion: batch/v1
kind: Job
metadata:
  name: batch-processor
spec:
  completions: 100
  parallelism: 10
  completionMode: Indexed
  backoffLimitPerIndex: 3    # 每个 index 最多重试 3 次
  maxFailedIndexes: 10       # 超过 10 个 index 失败后，整个 Job 失败
  template:
    spec:
      containers:
      - name: processor
        image: myapp/processor:v1
        env:
        - name: JOB_COMPLETION_INDEX
          valueFrom:
            fieldRef:
              fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
      restartPolicy: Never

apiVersion: batch/v1
kind: Job
metadata:
  name: ml-training
spec:
  backoffLimit: 6
  podFailurePolicy:
    rules:
    # OOM 导致的失败：立即终止整个 Job，不重试（资源不足）
    - action: FailJob
      onPodConditions:
      - type: DisruptionTarget
    # 退出码 42 表示数据错误：忽略该次失败，不计入 backoffLimit
    - action: Ignore
      onExitCodes:
        containerName: trainer
        operator: In
        values: [42]
    # 退出码 1：正常计入重试
    - action: Count
      onExitCodes:
        containerName: trainer
        operator: In
        values: [1]
  template:
    spec:
      containers:
      - name: trainer
        image: myapp/trainer:v1
      restartPolicy: Never

apiVersion: v1
kind: Pod
metadata:
  name: gpu-workload
  namespace: ml-team
spec:
  schedulingGates:
  - name: "example.com/gpu-quota-reserved"   # 自定义 gate 名称
  - name: "example.com/dataset-ready"
  containers:
  - name: trainer
    image: pytorch/pytorch:2.2
    resources:
      limits:
        nvidia.com/gpu: "4"

# 移除单个 gate
kubectl patch pod gpu-workload -n ml-team --type=json -p='[
  {
    "op": "remove",
    "path": "/spec/schedulingGates/0"
  }
]'

# 当所有 gate 都被移除后，Pod 才进入调度队列

apiVersion: groupsnapshot.storage.k8s.io/v1beta1
kind: VolumeGroupSnapshot
metadata:
  name: mysql-consistent-snapshot
  namespace: production
spec:
  volumeGroupSnapshotClassName: csi-aws-vgs-class
  source:
    selector:
      matchLabels:
        app: mysql
        snapshot-group: data-and-log   # 选择同一组的多个 PVC
---
# 对应的 PVC 需要打上相同标签
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql-data
  labels:
    app: mysql
    snapshot-group: data-and-log
spec:
  accessModes: [ReadWriteOnce]
  resources:
    requests:
      storage: 100Gi
  storageClassName: gp3-encrypted

apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
    - secrets
    - configmaps
    providers:
    - kms:
        apiVersion: v2          # 使用 KMS v2
        name: aws-kms-provider
        endpoint: unix:///var/run/kmsplugin/socket.sock
        timeout: 3s
        cachesize: 1000         # 本地缓存 DEK 数量
    - identity: {}              # 兜底：未加密读取（用于迁移）

- --encryption-provider-config=/etc/kubernetes/encryption-config.yaml
- --encryption-provider-config-automatic-reload=true  # 支持热重载配置

# 1. 先部署支持 v2 的 KMS plugin（兼容 v2 协议）
# 2. 更新 encryption-config，apiVersion 改为 v2
# 3. 执行 Secret 重加密（用新密钥重写所有 Secret）
kubectl get secrets --all-namespaces -o json | \
  kubectl replace -f -

# 4. 验证加密状态
kubectl get --raw='/healthz/etcd-encryption'

# /etc/kubernetes/kubelet-config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
memorySwap:
  swapBehavior: LimitedSwap
featureGates:
  NodeSwap: true

apiVersion: apiserver.config.k8s.io/v1alpha1
kind: AuthorizationConfiguration
authorizers:
  # 第一步：节点授权（kubelet 访问自己节点的资源）
  - type: Node
    name: node

  # 第二步：RBAC（标准权限检查）
  - type: RBAC
    name: rbac

  # 第三步：外部 OPA webhook（仅对特定资源调用）
  - type: Webhook
    name: opa-authz
    webhook:
      timeout: 3s
      failurePolicy: Deny          # webhook 不可用时拒绝请求
      matchConditions:
        # CEL 过滤：只有操作 secrets 或 rolebindings 才调用 OPA
        - expression: >
            request.resourceAttributes.resource in ['secrets', 'rolebindings']
      connectionInfo:
        type: KubeConfigFile
        kubeConfigFile: /etc/kubernetes/opa-authz-kubeconfig.yaml
      authorizedTTL: 5m            # 授权结果缓存 5 分钟
      unauthorizedTTL: 30s

- --authorization-config=/etc/kubernetes/authz-config.yaml
# 移除旧参数
# --authorization-mode=Node,RBAC  (由配置文件替代)

# 检查集群中是否还在使用废弃 API
kubectl get --raw /metrics | grep apiserver_requested_deprecated_apis

# 使用 pluto 扫描 YAML 文件
pluto detect-files -d ./k8s-manifests --target-versions k8s=v1.33.0

# 使用 kubent（kube-no-trouble）
kubent --target-version 1.33

# 1. 备份 etcd
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-$(date +%Y%m%d).db

# 2. 升级 control plane（以 kubeadm 为例）
kubeadm upgrade plan v1.33.0
kubeadm upgrade apply v1.33.0

# 3. 逐节点 drain + 升级 kubelet/kubectl
kubectl drain node-01 --ignore-daemonsets --delete-emptydir-data
apt-get install -y kubelet=1.33.0-* kubectl=1.33.0-*
systemctl restart kubelet
kubectl uncordon node-01

# 4. 验证集群健康
kubectl get nodes
kubectl get pods -A | grep -v Running | grep -v Completed