Skip to main content

5.2.1.4. Static Pods Setup

5.2.1.4.1. Kubernetes​

Static Pods setup

● Required

This section describes the manual creation of static pod manifests for Kubernetes control plane components.

Kube-API setup

● Required

Note

This section is optional and intended only for cases where this resource needs to be configured separately from the others.

Environment variables

export MACHINE_LOCAL_ADDRESS=$(ip -4 addr show scope global | awk '/inet/ {print $2; exit}' | cut -d/ -f1)

Working directory

mkdir -p /etc/kubernetes/manifests
Static Pod Kube-apiserver

Manifest generation

cat <<EOF > /etc/kubernetes/manifests/kube-apiserver.yaml
---
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: ${MACHINE_LOCAL_ADDRESS}:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=${MACHINE_LOCAL_ADDRESS}
    - --aggregator-reject-forwarding-redirect=true
    - --allow-privileged=true
    - --anonymous-auth=true
    - --api-audiences=konnectivity-server
    - --apiserver-count=1
    - --audit-log-batch-buffer-size=10000
    - --audit-log-batch-max-size=1
    - --audit-log-batch-max-wait=0s
    - --audit-log-batch-throttle-burst=0
    - --audit-log-batch-throttle-enable=false
    - --audit-log-batch-throttle-qps=0
    - --audit-log-compress=false
    - --audit-log-format=json
    - --audit-log-maxage=30
    - --audit-log-maxbackup=10
    - --audit-log-maxsize=1000
    - --audit-log-mode=batch
    - --audit-log-truncate-enabled=false
    - --audit-log-truncate-max-batch-size=10485760
    - --audit-log-truncate-max-event-size=102400
    - --audit-log-version=audit.k8s.io/v1
    - --audit-webhook-batch-buffer-size=10000
    - --audit-webhook-batch-initial-backoff=10s
    - --audit-webhook-batch-max-size=400
    - --audit-webhook-batch-max-wait=30s
    - --audit-webhook-batch-throttle-burst=15
    - --audit-webhook-batch-throttle-enable=true
    - --audit-webhook-batch-throttle-qps=10
    - --audit-webhook-initial-backoff=10s
    - --audit-webhook-mode=batch
    - --audit-webhook-truncate-enabled=false
    - --audit-webhook-truncate-max-batch-size=10485760
    - --audit-webhook-truncate-max-event-size=102400
    - --audit-webhook-version=audit.k8s.io/v1
    - --audit-policy-file=/etc/kubernetes/audit-policy.yaml
    - --audit-log-path=/var/log/kubernetes/audit/audit.log
    - --authentication-token-webhook-cache-ttl=2m0s
    - --authentication-token-webhook-version=v1beta1
    - --authorization-mode=Node,RBAC
    - --authorization-webhook-cache-authorized-ttl=5m0s
    - --authorization-webhook-cache-unauthorized-ttl=30s
    - --authorization-webhook-version=v1beta1
    - --bind-address=0.0.0.0
    - --cert-dir=/var/run/kubernetes
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    # -> Enable if managing state via Cloud Controller Manager
    # - --cloud-provider=external
    - --cloud-provider-gce-l7lb-src-cidrs=130.211.0.0/22,35.191.0.0/16
    - --cloud-provider-gce-lb-src-cidrs=130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16
    - --contention-profiling=false
    - --default-not-ready-toleration-seconds=300
    - --default-unreachable-toleration-seconds=300
    - --default-watch-cache-size=100
    - --delete-collection-workers=1
    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,PodSecurity
    - --enable-aggregator-routing=true
    - --enable-bootstrap-token-auth=true
    - --enable-garbage-collector=true
    - --enable-logs-handler=true
    - --enable-priority-and-fairness=true
    - --encryption-provider-config-automatic-reload=false
    - --endpoint-reconciler-type=lease
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-compaction-interval=5m0s
    - --etcd-count-metric-poll-period=1m0s
    - --etcd-db-metric-poll-interval=30s
    - --etcd-healthcheck-timeout=2s
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-prefix=/registry
    - --etcd-readycheck-timeout=2s
    - --etcd-servers=https://127.0.0.1:2379
    - --event-ttl=1h0m0s
    - --feature-gates=RotateKubeletServerCertificate=true
    - --goaway-chance=0
    - --help=false
    - --http2-max-streams-per-connection=0
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-port=10250
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --kubelet-read-only-port=10255
    - --kubelet-timeout=5s
    - --kubernetes-service-node-port=0
    - --lease-reuse-duration-seconds=60
    - --livez-grace-period=0s
    - --log-flush-frequency=5s
    - --logging-format=text
    - --log-json-info-buffer-size=0
    - --log-json-split-stream=false
    - --log-text-info-buffer-size=0
    - --log-text-split-stream=false
    - --max-connection-bytes-per-sec=0
    - --max-mutating-requests-inflight=200
    - --max-requests-inflight=400
    - --min-request-timeout=1800
    - --permit-address-sharing=false
    - --permit-port-sharing=false
    - --profiling=false
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --request-timeout=1m0s
    - --runtime-config=api/all=true
    - --secure-port=6443
    - --service-account-extend-token-expiration=true
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-account-lookup=true
    - --service-account-max-token-expiration=0s
    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=29.64.0.0/16
    - --service-node-port-range=30000-32767
    - --shutdown-delay-duration=0s
    - --shutdown-send-retry-after=false
    - --shutdown-watch-termination-grace-period=0s
    - --storage-backend=etcd3
    - --storage-media-type=application/vnd.kubernetes.protobuf
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    - --v=2
    - --version=false
    - --watch-cache=true
    # IF YOU NEED TO CONNECT CLOUD-CONTROLLER-MANAGER
    # UNCOMMENT THE FOLLOWING
    # ->
    # - --cloud-provider: "external"
    # Do not specify if value is "" or undefined
    # - --cloud-config=
    # - --strict-transport-security-directives=
    # - --disable-admission-plugins=
    # - --disabled-metrics=
    # - --egress-selector-config-file=
    # - --encryption-provider-config=
    # - --etcd-servers-overrides=
    # - --external-hostname=
    # - --kubelet-certificate-authority=
    # - --oidc-ca-file=
    # - --oidc-client-id=
    # - --oidc-groups-claim=
    # - --oidc-groups-prefix=
    # - --oidc-issuer-url=
    # - --oidc-required-claim=
    # - --oidc-signing-algs=RS256
    # - --oidc-username-claim=sub
    # - --oidc-username-prefix=
    # - --peer-advertise-ip=
    # - --peer-advertise-port=
    # - --peer-ca-file=
    # - --service-account-jwks-uri=
    # - --show-hidden-metrics-for-version=
    # - --tls-cipher-suites=
    # - --tls-min-version=
    # - --tls-sni-cert-key=
    # - --token-auth-file=
    # - --tracing-config-file=
    # - --vmodule=
    # - --watch-cache-sizes=
    # - --authorization-webhook-config-file=
    # - --cors-allowed-origins=
    # - --debug-socket-path=
    # - --authorization-policy-file=
    # - --authorization-config=
    # - --authentication-token-webhook-config-file=
    # - --authentication-config=
    # - --audit-webhook-config-file=
    # - --allow-metric-labels=
    # - --allow-metric-labels-manifest=
    # - --admission-control=
    # - --admission-control-config-file=
    # - --advertise-address=
    image: registry.k8s.io/kube-apiserver:v1.30.4
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: ${MACHINE_LOCAL_ADDRESS}
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-apiserver
    readinessProbe:
      failureThreshold: 3
      httpGet:
        host: ${MACHINE_LOCAL_ADDRESS}
        path: /readyz
        port: 6443
        scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 250m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: ${MACHINE_LOCAL_ADDRESS}
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
      readOnly: true
    - mountPath: /etc/pki
      name: etc-pki
      readOnly: true
    - mountPath: /var/log/kubernetes/audit/
      name: k8s-audit
    - mountPath: /etc/kubernetes/audit-policy.yaml
      name: k8s-audit-policy
      readOnly: true
    - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
      readOnly: true
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
      readOnly: true
  hostNetwork: true
  priority: 2000001000
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/ca-certificates
      type: DirectoryOrCreate
    name: etc-ca-certificates
  - hostPath:
      path: /etc/pki
      type: DirectoryOrCreate
    name: etc-pki
  - hostPath:
      path: /var/log/kubernetes/audit/
      type: DirectoryOrCreate
    name: k8s-audit
  - hostPath:
      path: /etc/kubernetes/audit-policy.yaml
      type: File
    name: k8s-audit-policy
  - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /usr/local/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-local-share-ca-certificates
  - hostPath:
      path: /usr/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-share-ca-certificates
status: {}
EOF

5.2.1.4.2. ETCD​

This section describes the manual creation of static pod manifests for ETCD.

Static Pods setup

● Required

Note

This section is optional and is intended only for cases when you need to configure this resource separately from the others.

Environment variables

export HOST_NAME=master-1
export CLUSTER_NAME=my-first-cluster
export BASE_DOMAIN=example.com
export MACHINE_LOCAL_ADDRESS=$(ip -4 addr show scope global | awk '/inet/ {print $2; exit}' | cut -d/ -f1)
export FULL_HOST_NAME="${HOST_NAME}.${CLUSTER_NAME}.${BASE_DOMAIN}"
export ETCD_INITIAL_CLUSTER="${FULL_HOST_NAME}=https://${MACHINE_LOCAL_ADDRESS}:2380"

Working directory

mkdir -p /etc/kubernetes/manifests
Static Pod ETCD

Manifest generation

cat <<EOF > /etc/kubernetes/manifests/etcd.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/etcd.advertise-client-urls: https://${MACHINE_LOCAL_ADDRESS}:2379
  creationTimestamp: null
  labels:
    component: etcd
    tier: control-plane
  name: etcd
  namespace: kube-system
spec:
  containers:
  - command:
    - etcd
    - --advertise-client-urls=https://${MACHINE_LOCAL_ADDRESS}:2379
    - --auto-compaction-retention=8
    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
    - --election-timeout=1500
    - --experimental-initial-corrupt-check=true
    - --experimental-watch-progress-notify-interval=5s
    - --heartbeat-interval=250
    - --initial-advertise-peer-urls=https://${MACHINE_LOCAL_ADDRESS}:2380
    - --initial-cluster=${ETCD_INITIAL_CLUSTER}
    - --key-file=/etc/kubernetes/pki/etcd/server.key
    - --listen-client-urls=https://0.0.0.0:2379
    - --listen-metrics-urls=http://0.0.0.0:2381
    - --listen-peer-urls=https://0.0.0.0:2380
    - --logger=zap
    - --max-snapshots=10
    - --max-wals=10
    - --metrics=extensive
    - --name=${FULL_HOST_NAME}
    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
    - --peer-client-cert-auth=true
    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    - --snapshot-count=10000
    - --quota-backend-bytes=10737418240
    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    image: registry.k8s.io/etcd:3.5.12-0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /health?exclude=NOSPACE&serializable=true
        port: 2381
        scheme: HTTP
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: etcd
    resources:
      requests:
        cpu: 100m
        memory: 100Mi
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 127.0.0.1
        path: /health?serializable=false
        port: 2381
        scheme: HTTP
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /var/lib/etcd
      name: etcd-data
    - mountPath: /etc/kubernetes/pki/etcd
      name: etcd-certs
  hostNetwork: true
  priority: 2000001000
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/kubernetes/pki/etcd
      type: DirectoryOrCreate
    name: etcd-certs
  - hostPath:
      path: /var/lib/etcd
      type: DirectoryOrCreate
    name: etcd-data
status: {}
EOF