5.2.1.3. Настройка ПО
В этом разделе описан процесс настройка и конфигурация компонентов Kubernetes, которые обеспечивают корректную работу кластера.
- containerd
- kubelet
- crictl
- kubeadm
- Kubernetes Audit
Настройка containerd
● Обязателен к применению
Настройка containerd
● Обязателен к применению
Этапы настройки компонента
- Конфигурация компонента
- Настройка Systemd Unit компонента
- Старт Systemd Unit
Данный раздел зависит от следующих документов:
Конфигурация компонента
- Bash
- Cloud-init
Создание рабочих директорий
mkdir -p /etc/containerd/
mkdir -p /etc/containerd/conf.d
mkdir -p /etc/containerd/certs.d
Базовый конфигурационный файл
cat <<"EOF" > /etc/containerd/config.toml
version = 2
imports = ["/etc/containerd/conf.d/*.toml"]
EOF
Шаблон кастомного конфигурационного файла
cat <<"EOF" > /etc/containerd/conf.d/in-cloud.toml
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.9"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d/"
EOF
Базовый конфигурационный файл
- path: /etc/containerd/config.toml
owner: root:root
permissions: '0644'
content: |
version = 2
imports = ["/etc/containerd/conf.d/*.toml"]
Шаблон кастомного конфигурационного файла
- path: /etc/containerd/conf.d/in-cloud.toml
owner: root:root
permissions: '0644'
content: |
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.9"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d/"
Настройка Systemd Unit компонента
- Bash
- Cloud-init
cat <<EOF > /usr/lib/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target containerd-install.service runc-install.service
Wants=containerd-install.service runc-install.service
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF
systemctl enable containerd
systemctl start containerd
# write_files:
- path: /usr/lib/systemd/system/containerd.service
owner: root:root
permissions: '0644'
content: |
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target containerd-install.service runc-install.service
Wants=containerd-install.service runc-install.service
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
#runcmd:
- systemctl enable containerd
- systemctl start containerd
Проверка настройки
Проверка настройки
tree /etc/containerd/
/etc/containerd/
├── certs.d
├── conf.d
│ └── cloud.toml
└── config.toml
systemctl status containerd
● containerd.service - containerd container runtime
Loaded: loaded (/usr/lib/systemd/system/containerd.service; enabled; preset: enabled)
Active: active (running) since Tue 2024-12-31 17:26:21 UTC; 2min 30s ago
Docs: https://containerd.io
Main PID: 839 (containerd)
Tasks: 7 (limit: 2274)
Memory: 62.0M (peak: 62.5M)
CPU: 375ms
CGroup: /system.slice/containerd.service
└─839 /usr/local/bin/containerd
***** level=info msg="Start subscribing containerd event"
***** level=info msg="Start recovering state"
***** level=info msg="Start event monitor"
***** level=info msg="Start snapshots syncer"
***** level=info msg="Start cni network conf syncer for default"
***** level=info msg="Start streaming server"
***** level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
***** level=info msg=serving... address=/run/containerd/containerd.sock
***** level=info msg="containerd successfully booted in 0.065807s"
***** Started containerd.service - containerd container runtime.
Настройка kubelet
● Обязателен к применению
Настройка kubelet
● Обязателен к применению
Этапы настройки компонента
- Настройка Systemd Unit компонента
- Добавить в автозагрузку Systemd Unit
- Кастомная конфигурация компонента
Данный раздел зависит от следующих документов:
Настройка Systemd Unit компонента
- Bash
- Cloud-init
mkdir -p /usr/lib/systemd/system/kubelet.service.d
mkdir -p /var/lib/kubelet/
Systemd Unit
cat <<EOF > /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target containerd.service
After=network-online.target containerd.service
[Service]
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
Systemd Unit Config
cat <<EOF > /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet/extra-args.env
ExecStart=
ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_KUBEADM_ARGS \$KUBELET_EXTRA_ARGS
EOF
- Default
- Cloud Controller Manager
Systemd Unit ENV
cat <<EOF > /etc/default/kubelet/extra-args.env
KUBELET_EXTRA_ARGS=""
EOF
Systemd Unit ENV
cat <<EOF > /etc/default/kubelet/extra-args.env
KUBELET_EXTRA_ARGS="--cloud-provider=external"
EOF
Добавить в автозагрузку Systemd Unit
systemctl enable kubelet
Systemd Unit ENV
# write_files:
- path: /usr/lib/systemd/system/kubelet.service
owner: root:root
permissions: '0644'
content: |
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target containerd.service
After=network-online.target containerd.service
[Service]
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
Инструкция загрузки
# write_files:
- path: /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
owner: root:root
permissions: '0644'
content: |
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet/extra-args.env
ExecStart=
ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
Systemd Unit Download
# write_files:
- path: /etc/default/kubelet/extra-args.env
owner: root:root
permissions: '0644'
content: |
KUBELET_EXTRA_ARGS="--cloud-provider=external"
Systemd Unit Custom ENV
Данный блок конфигурации применим только при установке Kubernetes вручную (методом «Kubernetes the Hard Way»). При использовании утилиты kubeadm конфигурационный файл будет создан автоматически на основе спецификации, указанной в файле kubeadm-config
.
# write_files:
- path: /var/lib/kubelet/kubeadm-flags.env
owner: root:root
permissions: '0644'
content: |
KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:3.9 --config=/var/lib/kubelet/config-custom.yaml"
Добавить в автозагрузку Systemd Unit
# runcmd:
- systemctl enable kubelet
Кастомная конфигурация компонента
Kubelet config
- Bash
- Cloud-init
Кастомный конфигурационный файл kubelet
cat <<EOF > /var/lib/kubelet/config-custom.yaml
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: "/etc/kubernetes/pki/ca.crt"
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
containerLogMaxSize: "50Mi"
containerRuntimeEndpoint: "/var/run/containerd/containerd.sock"
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 5s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageGCHighThresholdPercent: 55
imageGCLowThresholdPercent: 50
imageMaximumGCAge: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
text:
infoBufferSize: "0"
verbosity: 0
kubeAPIQPS: 50
kubeAPIBurst: 100
maxPods: 250
memorySwap: {}
nodeStatusReportFrequency: 1s
nodeStatusUpdateFrequency: 1s
podPidsLimit: 4096
registerNode: true
resolvConf: /run/systemd/resolve/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 0s
serializeImagePulls: false
serverTLSBootstrap: true
shutdownGracePeriod: 15s
shutdownGracePeriodCriticalPods: 5s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
tlsMinVersion: "VersionTLS12"
volumeStatsAggPeriod: 0s
featureGates:
RotateKubeletServerCertificate: true
APIPriorityAndFairness: true
tlsCipherSuites:
- "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
- "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
- "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
- "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
- "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
- "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"
EOF
Кастомный конфигурационный файл kubelet
- path: /var/lib/kubelet/config-custom.yaml
owner: root:root
permissions: '0644'
content: |
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: "/etc/kubernetes/pki/ca.crt"
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
containerLogMaxSize: "50Mi"
containerRuntimeEndpoint: "/var/run/containerd/containerd.sock"
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 5s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageGCHighThresholdPercent: 55
imageGCLowThresholdPercent: 50
imageMaximumGCAge: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
text:
infoBufferSize: "0"
verbosity: 0
kubeAPIQPS: 50
kubeAPIBurst: 100
maxPods: 250
memorySwap: {}
nodeStatusReportFrequency: 1s
nodeStatusUpdateFrequency: 1s
podPidsLimit: 4096
registerNode: true
resolvConf: /run/systemd/resolve/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 0s
serializeImagePulls: false
serverTLSBootstrap: true
shutdownGracePeriod: 15s
shutdownGracePeriodCriticalPods: 5s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
tlsMinVersion: "VersionTLS12"
volumeStatsAggPeriod: 0s
featureGates:
RotateKubeletServerCertificate: true
APIPriorityAndFairness: true
tlsCipherSuites:
- "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
- "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
- "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
- "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
- "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
- "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"
Проверка настройки
Проверка настройки
Обратите внимание, что при создании кластера с помощью Kubeadm без выполнения команд kubeadm init
или kubeadm join
, файл конфигурации Kubelet (/var/lib/kubelet/config.yaml
) не будет создан.
ls -la /var/lib/kubelet/config-custom.yaml
ls -la /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
-rw-r--r-- 1 root root 1721 Feb 19 18:57 /var/lib/kubelet/config.yaml
-rw-r--r-- 1 root root 903 Feb 19 22:10 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
Обратите внимание, что при создании кластера с помощью Kubeadm без выполнения команд kubeadm init
или kubeadm join
, Systemd Unit будет добавлен в автозапуск, но будет выключен.
systemctl status kubelet
○ kubelet.service - kubelet: The Kubernetes Node Agent
Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; preset: enabled)
Drop-In: /usr/lib/systemd/system/kubelet.service.d
└─10-kubeadm.conf
Active: inactive (dead)
Docs: https://kubernetes.io/docs/
Настройка crictl
● Не обязателен к применению
Настройка crictl
● Не обязателен к применению
Конфигурация компонента
Этапы настройки компонента
- Конфигурация компонента
- Настройка Systemd Unit компонента
- Старт Systemd Unit
Данный раздел зависит от следующих документов:
Конфигурация компонента
- Bash
- Cloud-init
Шаблон кастомного конфигурационно файла
cat <<"EOF" > /etc/crictl.yaml
runtime-endpoint: unix:///var/run/containerd/containerd.sock
EOF
Шаблон кастомного конфигурационно файла
- path: /etc/crictl.yaml
owner: root:root
permissions: '0644'
content: |
runtime-endpoint: unix:///var/run/containerd/containerd.sock
Проверка настройки
Проверка настройки
ls -la /etc/crictl.yaml
-rw-r--r-- 1 root root 61 Feb 18 15:18 /etc/crictl.yaml
crictl info |
jq '.status.conditions[] |
select(.type=="RuntimeReady") |
.status'
true
Настройка kubeadm
● Не обязателен к применению
Настройка kubeadm
● Не обязателен к применению
Этапы настройки компонента
- Создание рабочих директорий.
- Конфигурация компонента
Данный раздел зависит от следующих документов:
Создание рабочих директорий
mkdir -p /var/run/kubeadm/
Конфигурация компонента
- init
- join
Конфигурация Kubeadm
- master-1
export HOST_NAME=master-1
Конфигурация Kubeadm для инициализации кластера
- Bash
- Cloud-init
Переменные окружения для шаблона конфигурационного файла
export CLUSTER_NAME='my-first-cluster'
export BASE_DOMAIN='example.com'
export FULL_HOST_NAME=${HOST_NAME}.${CLUSTER_NAME}.${BASE_DOMAIN}
export INTERNAL_API=api.${CLUSTER_NAME}.${BASE_DOMAIN}
export MACHINE_LOCAL_ADDRESS=$(ip -4 addr show scope global | awk '/inet/ {print $2; exit}' | cut -d/ -f1)
export ETCD_INITIAL_CLUSTER="${FULL_HOST_NAME}=https://${MACHINE_LOCAL_ADDRESS}:2380"
export CERTIFICATE_UPLOAD_KEY=0c00c2fd5c67c37656c00d78a9d7e1f2eb794ef8e4fc3e2a4b532eb14323cd59
Конфигурационный файл kubeadm для инициализации кластера
Обратите внимание, что в данном конфигурационном файле этап установки addons
игнорируется.
cat <<EOF > /var/run/kubeadm/kubeadm.yaml
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
skipPhases:
- addon
bootstrapTokens:
- token: "fjt9ex.lwzqgdlvoxtqk4yw"
description: "kubeadm bootstrap token"
ttl: "24h"
certificateKey: ${CERTIFICATE_UPLOAD_KEY}
nodeRegistration:
imagePullPolicy: IfNotPresent
taints: null
kubeletExtraArgs:
# -> Включить, если управляете состоянием через Cloud Controller Manager
# cloud-provider: external
config: "/var/lib/kubelet/config-custom.yaml"
cluster-domain: cluster.local
cluster-dns: "29.64.0.10"
# name: '${FULL_HOST_NAME}'
ignorePreflightErrors:
# > При поэтапной сборке кластера, а не выполнении единой команды,
# > необходимо указать исключения в параметре ignorePreflightErrors,
# > чтобы команда kubeadm init phase preflight выполнялась без препятствий.
# > Для этого в nodeRegistration добавляются следующие исключения:
- FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml
- FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml
- FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml
- FileAvailable--etc-kubernetes-manifests-etcd.yaml
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
clusterName: "${CLUSTER_NAME}"
certificatesDir: /etc/kubernetes/pki
controlPlaneEndpoint: ${INTERNAL_API}:6443
imageRepository: "registry.k8s.io"
networking:
serviceSubnet: 29.64.0.0/16
dnsDomain: cluster.local
kubernetesVersion: v1.30.4
dns: {}
etcd:
local:
imageRepository: "registry.k8s.io"
dataDir: "/var/lib/etcd"
extraArgs:
auto-compaction-retention: "8"
cert-file: "/etc/kubernetes/pki/etcd/server.crt"
client-cert-auth: "true"
data-dir: "/var/lib/etcd"
election-timeout: "1500"
heartbeat-interval: "250"
key-file: "/etc/kubernetes/pki/etcd/server.key"
listen-client-urls: "https://0.0.0.0:2379"
listen-metrics-urls: "http://0.0.0.0:2381"
listen-peer-urls: "https://0.0.0.0:2380"
logger: "zap"
max-snapshots: "10"
max-wals: "10"
metrics: "extensive"
peer-cert-file: "/etc/kubernetes/pki/etcd/peer.crt"
peer-client-cert-auth: "true"
peer-key-file: "/etc/kubernetes/pki/etcd/peer.key"
peer-trusted-ca-file: "/etc/kubernetes/pki/etcd/ca.crt"
snapshot-count: "10000"
quota-backend-bytes: "10737418240" # TODO
experimental-initial-corrupt-check: "true"
experimental-watch-progress-notify-interval: "5s"
trusted-ca-file: "/etc/kubernetes/pki/etcd/ca.crt"
peerCertSANs:
- 127.0.0.1
serverCertSANs:
- 127.0.0.1
apiServer:
extraArgs:
aggregator-reject-forwarding-redirect: "true"
allow-privileged: "true"
anonymous-auth: "true"
api-audiences: "konnectivity-server"
apiserver-count: "1"
audit-log-batch-buffer-size: "10000"
audit-log-batch-max-size: "1"
audit-log-batch-max-wait: "0s"
audit-log-batch-throttle-burst: "0"
audit-log-batch-throttle-enable: "false"
audit-log-batch-throttle-qps: "0"
audit-log-compress: "false"
audit-log-format: "json"
audit-log-maxage: "30"
audit-log-maxbackup: "10"
audit-log-maxsize: "1000"
audit-log-mode: "batch"
audit-log-truncate-enabled: "false"
audit-log-truncate-max-batch-size: "10485760"
audit-log-truncate-max-event-size: "102400"
audit-log-version: "audit.k8s.io/v1"
audit-webhook-batch-buffer-size: "10000"
audit-webhook-batch-initial-backoff: "10s"
audit-webhook-batch-max-size: "400"
audit-webhook-batch-max-wait: "30s"
audit-webhook-batch-throttle-burst: "15"
audit-webhook-batch-throttle-enable: "true"
audit-webhook-batch-throttle-qps: "10"
audit-webhook-initial-backoff: "10s"
audit-webhook-mode: "batch"
audit-webhook-truncate-enabled: "false"
audit-webhook-truncate-max-batch-size: "10485760"
audit-webhook-truncate-max-event-size: "102400"
audit-webhook-version: "audit.k8s.io/v1"
audit-policy-file: /var/log/kubernetes/audit/audit-policy.yaml
audit-log-path: /var/log/kubernetes/audit/audit.log
authentication-token-webhook-cache-ttl: "2m0s"
authentication-token-webhook-version: "v1beta1"
authorization-mode: "Node,RBAC"
authorization-webhook-cache-authorized-ttl: "5m0s"
authorization-webhook-cache-unauthorized-ttl: "30s"
authorization-webhook-version: "v1beta1"
bind-address: "0.0.0.0"
cert-dir: "/var/run/kubernetes"
client-ca-file: "/etc/kubernetes/pki/ca.crt"
cloud-provider-gce-l7lb-src-cidrs: "130.211.0.0/22,35.191.0.0/16"
cloud-provider-gce-lb-src-cidrs: "130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16"
contention-profiling: "false"
default-not-ready-toleration-seconds: "300"
default-unreachable-toleration-seconds: "300"
default-watch-cache-size: "100"
delete-collection-workers: "1"
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,AlwaysPullImages,NodeRestriction,PodSecurity"
enable-aggregator-routing: "true"
enable-bootstrap-token-auth: "true"
enable-garbage-collector: "true"
enable-logs-handler: "true"
enable-priority-and-fairness: "true"
encryption-provider-config-automatic-reload: "false"
endpoint-reconciler-type: "lease"
etcd-cafile: "/etc/kubernetes/pki/etcd/ca.crt"
etcd-certfile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
etcd-compaction-interval: "5m0s"
etcd-count-metric-poll-period: "1m0s"
etcd-db-metric-poll-interval: "30s"
etcd-healthcheck-timeout: "2s"
etcd-keyfile: "/etc/kubernetes/pki/apiserver-etcd-client.key"
etcd-prefix: "/registry"
etcd-readycheck-timeout: "2s"
etcd-servers: "https://127.0.0.1:2379"
event-ttl: "1h0m0s"
feature-gates: "RotateKubeletServerCertificate=true"
goaway-chance: "0"
help: "false"
http2-max-streams-per-connection: "0"
kubelet-client-certificate: "/etc/kubernetes/pki/apiserver-kubelet-client.crt"
kubelet-client-key: "/etc/kubernetes/pki/apiserver-kubelet-client.key"
kubelet-port: "10250"
kubelet-preferred-address-types: "InternalIP,ExternalIP,Hostname"
kubelet-read-only-port: "10255"
kubelet-timeout: "5s"
kubernetes-service-node-port: "0"
lease-reuse-duration-seconds: "60"
livez-grace-period: "0s"
log-flush-frequency: "5s"
logging-format: "text"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
max-connection-bytes-per-sec: "0"
max-mutating-requests-inflight: "200"
max-requests-inflight: "400"
min-request-timeout: "1800"
permit-address-sharing: "false"
permit-port-sharing: "false"
profiling: "false"
proxy-client-cert-file: "/etc/kubernetes/pki/front-proxy-client.crt"
proxy-client-key-file: "/etc/kubernetes/pki/front-proxy-client.key"
requestheader-allowed-names: "front-proxy-client"
requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
requestheader-extra-headers-prefix: "X-Remote-Extra-"
requestheader-group-headers: "X-Remote-Group"
requestheader-username-headers: "X-Remote-User"
request-timeout: "1m0s"
runtime-config: "api/all=true"
secure-port: "6443"
service-account-extend-token-expiration: "true"
service-account-issuer: "https://kubernetes.default.svc.cluster.local"
service-account-key-file: "/etc/kubernetes/pki/sa.pub"
service-account-lookup: "true"
service-account-max-token-expiration: "0s"
service-account-signing-key-file: "/etc/kubernetes/pki/sa.key"
service-cluster-ip-range: "29.64.0.0/16"
service-node-port-range: "30000-32767"
shutdown-delay-duration: "0s"
shutdown-send-retry-after: "false"
shutdown-watch-termination-grace-period: "0s"
storage-backend: "etcd3"
storage-media-type: "application/vnd.kubernetes.protobuf"
tls-cert-file: "/etc/kubernetes/pki/apiserver.crt"
tls-private-key-file: "/etc/kubernetes/pki/apiserver.key"
v: "2"
version: "false"
watch-cache: "true"
# ЕСЛИ НУЖНО ПОДКЛЮЧИТЬ CLOUD-CONTROLLER-MANAGER
# ТРЕ БУЕТСЯ РАСКОМЕНТИРОВАТЬ
# ->
# cloud-provider: "external"
# Не указывать если значение "" или undefined
# cloud-config: ""
# strict-transport-security-directives: ""
# disable-admission-plugins: ""
# disabled-metrics: ""
# egress-selector-config-file: ""
# encryption-provider-config: ""
# etcd-servers-overrides: ""
# external-hostname: ""
# kubelet-certificate-authority: ""
# oidc-ca-file: ""
# oidc-client-id: ""
# oidc-groups-claim: ""
# oidc-groups-prefix: ""
# oidc-issuer-url: ""
# oidc-required-claim: ""
# oidc-signing-algs: "RS256"
# oidc-username-claim: "sub"
# oidc-username-prefix: ""
# peer-advertise-ip: ""
# peer-advertise-port: ""
# peer-ca-file: ""
# service-account-jwks-uri: ""
# show-hidden-metrics-for-version: ""
# tls-cipher-suites: ""
# tls-min-version: ""
# tls-sni-cert-key: ""
# token-auth-file: ""
# tracing-config-file: ""
# vmodule: ""
# watch-cache-sizes: ""
# authorization-webhook-config-file: ""
# cors-allowed-origins: ""
# debug-socket-path: ""
# authorization-policy-file: ""
# authorization-config: ""
# authentication-token-webhook-config-file: ""
# authentication-config: ""
# audit-webhook-config-file: ""
# audit-policy-file: "/var/log/kubernetes/audit/audit-policy.yaml"
# audit-log-path: "/var/log/kubernetes/audit/audit.log"
# allow-metric-labels: ""
# allow-metric-labels-manifest: ""
# admission-control: ""
# admission-control-config-file: ""
# advertise-address: ""
extraVolumes:
- name: "k8s-audit"
hostPath: "/var/log/kubernetes/audit/"
mountPath: "/var/log/kubernetes/audit/"
readOnly: false
pathType: DirectoryOrCreate
certSANs:
- "127.0.0.1"
# TODO для доабвления внешнего FQDN в сертификаты кластера
# - ${INTERNAL_API}
timeoutForControlPlane: 4m0s
controllerManager:
extraArgs:
cluster-name: "${CLUSTER_NAME}"
allocate-node-cidrs: "false"
allow-untagged-cloud: "false"
attach-detach-reconcile-sync-period: "1m0s"
authentication-kubeconfig: "/etc/kubernetes/controller-manager.conf"
authentication-skip-lookup: "false"
authentication-token-webhook-cache-ttl: "10s"
authentication-tolerate-lookup-failure: "false"
authorization-always-allow-paths: "/healthz,/readyz,/livez,/metrics"
authorization-kubeconfig: "/etc/kubernetes/controller-manager.conf"
authorization-webhook-cache-authorized-ttl: "10s"
authorization-webhook-cache-unauthorized-ttl: "10s"
bind-address: "0.0.0.0"
cidr-allocator-type: "RangeAllocator"
client-ca-file: "/etc/kubernetes/pki/ca.crt"
# -> Включить, если управляете состоянием через Cloud Controller Manager
# cloud-provider: "external"
cloud-provider-gce-lb-src-cidrs: "130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16"
cluster-signing-cert-file: "/etc/kubernetes/pki/ca.crt"
cluster-signing-duration: "720h0m0s"
cluster-signing-key-file: "/etc/kubernetes/pki/ca.key"
concurrent-cron-job-syncs: "5"
concurrent-deployment-syncs: "5"
concurrent-endpoint-syncs: "5"
concurrent-ephemeralvolume-syncs: "5"
concurrent-gc-syncs: "20"
concurrent-horizontal-pod-autoscaler-syncs: "5"
concurrent-job-syncs: "5"
concurrent-namespace-syncs: "10"
concurrent-rc-syncs: "5"
concurrent-replicaset-syncs: "20"
concurrent-resource-quota-syncs: "5"
concurrent-service-endpoint-syncs: "5"
concurrent-service-syncs: "1"
concurrent-serviceaccount-token-syncs: "5"
concurrent-statefulset-syncs: "5"
concurrent-ttl-after-finished-syncs: "5"
concurrent-validating-admission-policy-status-syncs: "5"
configure-cloud-routes: "true"
contention-profiling: "false"
controller-start-interval: "0s"
controllers: "*,bootstrapsigner,tokencleaner"
disable-attach-detach-reconcile-sync: "false"
disable-force-detach-on-timeout: "false"
enable-dynamic-provisioning: "true"
enable-garbage-collector: "true"
enable-hostpath-provisioner: "false"
enable-leader-migration: "false"
endpoint-updates-batch-period: "0s"
endpointslice-updates-batch-period: "0s"
feature-gates: "RotateKubeletServerCertificate=true"
flex-volume-plugin-dir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
help: "false"
horizontal-pod-autoscaler-cpu-initialization-period: "5m0s"
horizontal-pod-autoscaler-downscale-delay: "5m0s"
horizontal-pod-autoscaler-downscale-stabilization: "5m0s"
horizontal-pod-autoscaler-initial-readiness-delay: "30s"
horizontal-pod-autoscaler-sync-period: "30s"
horizontal-pod-autoscaler-tolerance: "0.1"
horizontal-pod-autoscaler-upscale-delay: "3m0s"
http2-max-streams-per-connection: "0"
kube-api-burst: "120"
kube-api-content-type: "application/vnd.kubernetes.protobuf"
kube-api-qps: "100"
kubeconfig: "/etc/kubernetes/controller-manager.conf"
large-cluster-size-threshold: "50"
leader-elect: "true"
leader-elect-lease-duration: "15s"
leader-elect-renew-deadline: "10s"
leader-elect-resource-lock: "leases"
leader-elect-resource-name: "kube-controller-manager"
leader-elect-resource-namespace: "kube-system"
leader-elect-retry-period: "2s"
legacy-service-account-token-clean-up-period: "8760h0m0s"
log-flush-frequency: "5s"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
logging-format: "text"
max-endpoints-per-slice: "100"
min-resync-period: "12h0m0s"
mirroring-concurrent-service-endpoint-syncs: "5"
mirroring-endpointslice-updates-batch-period: "0s"
mirroring-max-endpoints-per-subset: "1000"
namespace-sync-period: "2m0s"
node-cidr-mask-size: "0"
node-cidr-mask-size-ipv4: "0"
node-cidr-mask-size-ipv6: "0"
node-eviction-rate: "0.1"
node-monitor-grace-period: "40s"
node-monitor-period: "5s"
node-startup-grace-period: "10s"
node-sync-period: "0s"
permit-address-sharing: "false"
permit-port-sharing: "false"
profiling: "false"
pv-recycler-increment-timeout-nfs: "30"
pv-recycler-minimum-timeout-hostpath: "60"
pv-recycler-minimum-timeout-nfs: "300"
pv-recycler-timeout-increment-hostpath: "30"
pvclaimbinder-sync-period: "15s"
requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
requestheader-extra-headers-prefix: "x-remote-extra-"
requestheader-group-headers: "x-remote-group"
requestheader-username-headers: "x-remote-user"
resource-quota-sync-period: "5m0s"
root-ca-file: "/etc/kubernetes/pki/ca.crt"
route-reconciliation-period: "10s"
secondary-node-eviction-rate: "0.01"
secure-port: "10257"
service-account-private-key-file: "/etc/kubernetes/pki/sa.key"
terminated-pod-gc-threshold: "0"
unhealthy-zone-threshold: "0.55"
use-service-account-credentials: "true"
v: "2"
version: "false"
volume-host-allow-local-loopback: "true"
# Не указывать если значение "" или undefined
# cluster-signing-kube-apiserver-client-cert-file: ""
# cluster-signing-kube-apiserver-client-key-file: ""
# cluster-signing-kubelet-client-cert-file: ""
# cluster-signing-kubelet-client-key-file: ""
# cluster-signing-kubelet-serving-cert-file: ""
# cluster-signing-kubelet-serving-key-file: ""
# cluster-signing-legacy-unknown-cert-file: ""
# cluster-signing-legacy-unknown-key-file: ""
# cluster-cidr: ""
# cloud-config: ""
# cert-dir: ""
# allow-metric-labels-manifest: ""
# allow-metric-labels: ""
# disabled-metrics: ""
# leader-migration-config: ""
# master: ""
# pv-recycler-pod-template-filepath-hostpath: ""
# pv-recycler-pod-template-filepath-nfs: ""
# service-cluster-ip-range: ""
# show-hidden-metrics-for-version: ""
# tls-cert-file: ""
# tls-cipher-suites: ""
# tls-min-version: ""
# tls-private-key-file: ""
# tls-sni-cert-key: ""
# vmodule: ""
# volume-host-cidr-denylist: ""
# external-cloud-volume-plugin: ""
# requestheader-allowed-names: ""
scheduler:
extraArgs:
authentication-kubeconfig: "/etc/kubernetes/scheduler.conf"
authentication-skip-lookup: "false"
authentication-token-webhook-cache-ttl: "10s"
authentication-tolerate-lookup-failure: "true"
authorization-always-allow-paths: "/healthz,/readyz,/livez,/metrics"
authorization-kubeconfig: "/etc/kubernetes/scheduler.conf"
authorization-webhook-cache-authorized-ttl: "10s"
authorization-webhook-cache-unauthorized-ttl: "10s"
bind-address: "0.0.0.0"
client-ca-file: ""
contention-profiling: "true"
help: "false"
http2-max-streams-per-connection: "0"
kube-api-burst: "100"
kube-api-content-type: "application/vnd.kubernetes.protobuf"
kube-api-qps: "50"
kubeconfig: "/etc/kubernetes/scheduler.conf"
leader-elect: "true"
leader-elect-lease-duration: "15s"
leader-elect-renew-deadline: "10s"
leader-elect-resource-lock: "leases"
leader-elect-resource-name: "kube-scheduler"
leader-elect-resource-namespace: "kube-system"
leader-elect-retry-period: "2s"
log-flush-frequency: "5s"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
logging-format: "text"
permit-address-sharing: "false"
permit-port-sharing: "false"
pod-max-in-unschedulable-pods-duration: "5m0s"
profiling: "true"
requestheader-extra-headers-prefix: "[x-remote-extra-]"
requestheader-group-headers: "[x-remote-group]"
requestheader-username-headers: "[x-remote-user]"
secure-port: "10259"
v: "2"
version: "false"
# allow-metric-labels: "[]"
# allow-metric-labels-manifest: ""
# cert-dir: ""
# config: ""
# disabled-metrics: "[]"
# feature-gates: ""
# master: ""
# requestheader-allowed-names: "[]"
# requestheader-client-ca-file: ""
# show-hidden-metrics-for-version: ""
# tls-cert-file: ""
# tls-cipher-suites: "[]"
# tls-min-version: ""
# tls-private-key-file: ""
# tls-sni-cert-key: "[]"
# vmodule: ""
# write-config-to: ""
EOF
- path: /var/run/kubeadm/kubeadm.yaml
owner: root:root
permissions: '0644'
content: |
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
skipPhases:
- addon
bootstrapTokens:
- token: "fjt9ex.lwzqgdlvoxtqk4yw"
description: "kubeadm bootstrap token"
ttl: "24h"
certificateKey: 0c00c2fd5c67c37656c00d78a9d7e1f2eb794ef8e4fc3e2a4b532eb14323cd59
nodeRegistration:
imagePullPolicy: IfNotPresent
taints: null
kubeletExtraArgs:
cloud-provider: external
config: "/var/lib/kubelet/config-custom.yaml"
cluster-domain: cluster.local
cluster-dns: "29.64.0.10"
# TODO тут нужно подумать, нужно ли определение имени узла пр и инициализации УК
# name: {{ local_hostname }}
ignorePreflightErrors:
# > При поэтапной сборке кластера, а не выполнении единой команды,
# > необходимо указать исключения в параметре ignorePreflightErrors,
# > чтобы команда kubeadm init phase preflight выполнялась без препятствий.
# > Для этого в nodeRegistration добавляются следующие исключения:
- FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml
- FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml
- FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml
- FileAvailable--etc-kubernetes-manifests-etcd.yaml
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
clusterName: "my-first-cluster"
# TODO нужно указывать VIP LoadBalancer
controlPlaneEndpoint: {{ local_hostname }}:6443
imageRepository: "registry.k8s.io"
networking:
serviceSubnet: 29.64.0.0/16
dnsDomain: cluster.local
kubernetesVersion: v1.30.4
dns: {}
etcd:
local:
imageRepository: "registry.k8s.io"
dataDir: "/var/lib/etcd"
extraArgs:
auto-compaction-retention: "8"
cert-file: "/etc/kubernetes/pki/etcd/server.crt"
client-cert-auth: "true"
data-dir: "/var/lib/etcd"
election-timeout: "1500"
heartbeat-interval: "250"
key-file: "/etc/kubernetes/pki/etcd/server.key"
listen-client-urls: "https://0.0.0.0:2379"
listen-metrics-urls: "http://0.0.0.0:2381"
listen-peer-urls: "https://0.0.0.0:2380"
logger: "zap"
max-snapshots: "10"
max-wals: "10"
metrics: "extensive"
peer-cert-file: "/etc/kubernetes/pki/etcd/peer.crt"
peer-client-cert-auth: "true"
peer-key-file: "/etc/kubernetes/pki/etcd/peer.key"
peer-trusted-ca-file: "/etc/kubernetes/pki/etcd/ca.crt"
snapshot-count: "10000"
quota-backend-bytes: "10737418240" # TODO
experimental-initial-corrupt-check: "true"
experimental-watch-progress-notify-interval: "5s"
trusted-ca-file: "/etc/kubernetes/pki/etcd/ca.crt"
peerCertSANs:
- 127.0.0.1
serverCertSANs:
- 127.0.0.1
apiServer:
extraArgs:
aggregator-reject-forwarding-redirect: "true"
allow-privileged: "true"
anonymous-auth: "true"
api-audiences: "konnectivity-server"
apiserver-count: "1"
audit-log-batch-buffer-size: "10000"
audit-log-batch-max-size: "1"
audit-log-batch-max-wait: "0s"
audit-log-batch-throttle-burst: "0"
audit-log-batch-throttle-enable: "false"
audit-log-batch-throttle-qps: "0"
audit-log-compress: "false"
audit-log-format: "json"
audit-log-maxage: "30"
audit-log-maxbackup: "10"
audit-log-maxsize: "1000"
audit-log-mode: "batch"
audit-log-truncate-enabled: "false"
audit-log-truncate-max-batch-size: "10485760"
audit-log-truncate-max-event-size: "102400"
audit-log-version: "audit.k8s.io/v1"
audit-webhook-batch-buffer-size: "10000"
audit-webhook-batch-initial-backoff: "10s"
audit-webhook-batch-max-size: "400"
audit-webhook-batch-max-wait: "30s"
audit-webhook-batch-throttle-burst: "15"
audit-webhook-batch-throttle-enable: "true"
audit-webhook-batch-throttle-qps: "10"
audit-webhook-initial-backoff: "10s"
audit-webhook-mode: "batch"
audit-webhook-truncate-enabled: "false"
audit-webhook-truncate-max-batch-size: "10485760"
audit-webhook-truncate-max-event-size: "102400"
audit-webhook-version: "audit.k8s.io/v1"
audit-policy-file: /var/log/kubernetes/audit/audit-policy.yaml
audit-log-path: /var/log/kubernetes/audit/audit.log
authentication-token-webhook-cache-ttl: "2m0s"
authentication-token-webhook-version: "v1beta1"
authorization-mode: "Node,RBAC"
authorization-webhook-cache-authorized-ttl: "5m0s"
authorization-webhook-cache-unauthorized-ttl: "30s"
authorization-webhook-version: "v1beta1"
bind-address: "0.0.0.0"
cert-dir: "/var/run/kubernetes"
client-ca-file: "/etc/kubernetes/pki/ca.crt"
cloud-provider-gce-l7lb-src-cidrs: "130.211.0.0/22,35.191.0.0/16"
cloud-provider-gce-lb-src-cidrs: "130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16"
contention-profiling: "false"
default-not-ready-toleration-seconds: "300"
default-unreachable-toleration-seconds: "300"
default-watch-cache-size: "100"
delete-collection-workers: "1"
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,AlwaysPullImages,NodeRestriction,PodSecurity"
enable-aggregator-routing: "true"
enable-bootstrap-token-auth: "true"
enable-garbage-collector: "true"
enable-logs-handler: "true"
enable-priority-and-fairness: "true"
encryption-provider-config-automatic-reload: "false"
endpoint-reconciler-type: "lease"
etcd-cafile: "/etc/kubernetes/pki/etcd/ca.crt"
etcd-certfile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
etcd-compaction-interval: "5m0s"
etcd-count-metric-poll-period: "1m0s"
etcd-db-metric-poll-interval: "30s"
etcd-healthcheck-timeout: "2s"
etcd-keyfile: "/etc/kubernetes/pki/apiserver-etcd-client.key"
etcd-prefix: "/registry"
etcd-readycheck-timeout: "2s"
etcd-servers: "https://127.0.0.1:2379"
event-ttl: "1h0m0s"
feature-gates: "RotateKubeletServerCertificate=true"
goaway-chance: "0"
help: "false"
http2-max-streams-per-connection: "0"
kubelet-client-certificate: "/etc/kubernetes/pki/apiserver-kubelet-client.crt"
kubelet-client-key: "/etc/kubernetes/pki/apiserver-kubelet-client.key"
kubelet-port: "10250"
kubelet-preferred-address-types: "InternalIP,ExternalIP,Hostname"
kubelet-read-only-port: "10255"
kubelet-timeout: "5s"
kubernetes-service-node-port: "0"
lease-reuse-duration-seconds: "60"
livez-grace-period: "0s"
log-flush-frequency: "5s"
logging-format: "text"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
max-connection-bytes-per-sec: "0"
max-mutating-requests-inflight: "200"
max-requests-inflight: "400"
min-request-timeout: "1800"
permit-address-sharing: "false"
permit-port-sharing: "false"
profiling: "false"
proxy-client-cert-file: "/etc/kubernetes/pki/front-proxy-client.crt"
proxy-client-key-file: "/etc/kubernetes/pki/front-proxy-client.key"
requestheader-allowed-names: "front-proxy-client"
requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
requestheader-extra-headers-prefix: "X-Remote-Extra-"
requestheader-group-headers: "X-Remote-Group"
requestheader-username-headers: "X-Remote-User"
request-timeout: "1m0s"
runtime-config: "api/all=true"
secure-port: "6443"
service-account-extend-token-expiration: "true"
service-account-issuer: "https://kubernetes.default.svc.cluster.local"
service-account-key-file: "/etc/kubernetes/pki/sa.pub"
service-account-lookup: "true"
service-account-max-token-expiration: "0s"
service-account-signing-key-file: "/etc/kubernetes/pki/sa.key"
service-cluster-ip-range: "29.64.0.0/16"
service-node-port-range: "30000-32767"
shutdown-delay-duration: "0s"
shutdown-send-retry-after: "false"
shutdown-watch-termination-grace-period: "0s"
storage-backend: "etcd3"
storage-media-type: "application/vnd.kubernetes.protobuf"
tls-cert-file: "/etc/kubernetes/pki/apiserver.crt"
tls-private-key-file: "/etc/kubernetes/pki/apiserver.key"
v: "2"
version: "false"
watch-cache: "true"
# ЕСЛИ НУЖНО ПОДКЛЮЧИТЬ CLOUD-CONTROLLER-MANAGER
# ТРЕБУЕТСЯ РАСКОМЕНТИРОВАТЬ
# ->
# cloud-provider: "external"
# Не указывать если значение "" или undefined
# cloud-config: ""
# strict-transport-security-directives: ""
# disable-admission-plugins: ""
# disabled-metrics: ""
# egress-selector-config-file: ""
# encryption-provider-config: ""
# etcd-servers-overrides: ""
# external-hostname: ""
# kubelet-certificate-authority: ""
# oidc-ca-file: ""
# oidc-client-id: ""
# oidc-groups-claim: ""
# oidc-groups-prefix: ""
# oidc-issuer-url: ""
# oidc-required-claim: ""
# oidc-signing-algs: "RS256"
# oidc-username-claim: "sub"
# oidc-username-prefix: ""
# peer-advertise-ip: ""
# peer-advertise-port: ""
# peer-ca-file: ""
# service-account-jwks-uri: ""
# show-hidden-metrics-for-version: ""
# tls-cipher-suites: ""
# tls-min-version: ""
# tls-sni-cert-key: ""
# token-auth-file: ""
# tracing-config-file: ""
# vmodule: ""
# watch-cache-sizes: ""
# authorization-webhook-config-file: ""
# cors-allowed-origins: ""
# debug-socket-path: ""
# authorization-policy-file: ""
# authorization-config: ""
# authentication-token-webhook-config-file: ""
# authentication-config: ""
# audit-webhook-config-file: ""
# audit-policy-file: "/var/log/kubernetes/audit/audit-policy.yaml"
# audit-log-path: "/var/log/kubernetes/audit/audit.log"
# allow-metric-labels: ""
# allow-metric-labels-manifest: ""
# admission-control: ""
# admission-control-config-file: ""
# advertise-address: ""
extraVolumes:
- name: "k8s-audit"
hostPath: "/var/log/kubernetes/audit/"
mountPath: "/var/log/kubernetes/audit/"
pathType: DirectoryOrCreate
certSANs:
- "127.0.0.1"
timeoutForControlPlane: 4m0s
controllerManager:
extraArgs:
cluster-name: "my-first-cluster"
allocate-node-cidrs: "false"
allow-untagged-cloud: "false"
attach-detach-reconcile-sync-period: "1m0s"
authentication-kubeconfig: "/etc/kubernetes/controller-manager.conf"
authentication-skip-lookup: "false"
authentication-token-webhook-cache-ttl: "10s"
authentication-tolerate-lookup-failure: "false"
authorization-always-allow-paths: "/healthz,/readyz,/livez,/metrics"
authorization-kubeconfig: "/etc/kubernetes/controller-manager.conf"
authorization-webhook-cache-authorized-ttl: "10s"
authorization-webhook-cache-unauthorized-ttl: "10s"
bind-address: "0.0.0.0"
cidr-allocator-type: "RangeAllocator"
client-ca-file: "/etc/kubernetes/pki/ca.crt"
# -> Включить, если управляете состоянием через Cloud Controller Manager
# cloud-provider: "external"
cloud-provider-gce-lb-src-cidrs: "130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16"
cluster-signing-cert-file: "/etc/kubernetes/pki/ca.crt"
cluster-signing-duration: "720h0m0s"
cluster-signing-key-file: "/etc/kubernetes/pki/ca.key"
concurrent-cron-job-syncs: "5"
concurrent-deployment-syncs: "5"
concurrent-endpoint-syncs: "5"
concurrent-ephemeralvolume-syncs: "5"
concurrent-gc-syncs: "20"
concurrent-horizontal-pod-autoscaler-syncs: "5"
concurrent-job-syncs: "5"
concurrent-namespace-syncs: "10"
concurrent-rc-syncs: "5"
concurrent-replicaset-syncs: "20"
concurrent-resource-quota-syncs: "5"
concurrent-service-endpoint-syncs: "5"
concurrent-service-syncs: "1"
concurrent-serviceaccount-token-syncs: "5"
concurrent-statefulset-syncs: "5"
concurrent-ttl-after-finished-syncs: "5"
concurrent-validating-admission-policy-status-syncs: "5"
configure-cloud-routes: "true"
contention-profiling: "false"
controller-start-interval: "0s"
controllers: "*,bootstrapsigner,tokencleaner"
disable-attach-detach-reconcile-sync: "false"
disable-force-detach-on-timeout: "false"
enable-dynamic-provisioning: "true"
enable-garbage-collector: "true"
enable-hostpath-provisioner: "false"
enable-leader-migration: "false"
endpoint-updates-batch-period: "0s"
endpointslice-updates-batch-period: "0s"
feature-gates: "RotateKubeletServerCertificate=true"
flex-volume-plugin-dir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
help: "false"
horizontal-pod-autoscaler-cpu-initialization-period: "5m0s"
horizontal-pod-autoscaler-downscale-delay: "5m0s"
horizontal-pod-autoscaler-downscale-stabilization: "5m0s"
horizontal-pod-autoscaler-initial-readiness-delay: "30s"
horizontal-pod-autoscaler-sync-period: "30s"
horizontal-pod-autoscaler-tolerance: "0.1"
horizontal-pod-autoscaler-upscale-delay: "3m0s"
http2-max-streams-per-connection: "0"
kube-api-burst: "120"
kube-api-content-type: "application/vnd.kubernetes.protobuf"
kube-api-qps: "100"
kubeconfig: "/etc/kubernetes/controller-manager.conf"
large-cluster-size-threshold: "50"
leader-elect: "true"
leader-elect-lease-duration: "15s"
leader-elect-renew-deadline: "10s"
leader-elect-resource-lock: "leases"
leader-elect-resource-name: "kube-controller-manager"
leader-elect-resource-namespace: "kube-system"
leader-elect-retry-period: "2s"
legacy-service-account-token-clean-up-period: "8760h0m0s"
log-flush-frequency: "5s"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
logging-format: "text"
max-endpoints-per-slice: "100"
min-resync-period: "12h0m0s"
mirroring-concurrent-service-endpoint-syncs: "5"
mirroring-endpointslice-updates-batch-period: "0s"
mirroring-max-endpoints-per-subset: "1000"
namespace-sync-period: "2m0s"
node-cidr-mask-size: "0"
node-cidr-mask-size-ipv4: "0"
node-cidr-mask-size-ipv6: "0"
node-eviction-rate: "0.1"
node-monitor-grace-period: "40s"
node-monitor-period: "5s"
node-startup-grace-period: "10s"
node-sync-period: "0s"
permit-address-sharing: "false"
permit-port-sharing: "false"
profiling: "false"
pv-recycler-increment-timeout-nfs: "30"
pv-recycler-minimum-timeout-hostpath: "60"
pv-recycler-minimum-timeout-nfs: "300"
pv-recycler-timeout-increment-hostpath: "30"
pvclaimbinder-sync-period: "15s"
requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
requestheader-extra-headers-prefix: "x-remote-extra-"
requestheader-group-headers: "x-remote-group"
requestheader-username-headers: "x-remote-user"
resource-quota-sync-period: "5m0s"
root-ca-file: "/etc/kubernetes/pki/ca.crt"
route-reconciliation-period: "10s"
secondary-node-eviction-rate: "0.01"
secure-port: "10257"
service-account-private-key-file: "/etc/kubernetes/pki/sa.key"
terminated-pod-gc-threshold: "0"
unhealthy-zone-threshold: "0.55"
use-service-account-credentials: "true"
v: "2"
version: "false"
volume-host-allow-local-loopback: "true"
# Не указывать если значение "" или undefined
# cluster-signing-kube-apiserver-client-cert-file: ""
# cluster-signing-kube-apiserver-client-key-file: ""
# cluster-signing-kubelet-client-cert-file: ""
# cluster-signing-kubelet-client-key-file: ""
# cluster-signing-kubelet-serving-cert-file: ""
# cluster-signing-kubelet-serving-key-file: ""
# cluster-signing-legacy-unknown-cert-file: ""
# cluster-signing-legacy-unknown-key-file: ""
# cluster-cidr: ""
# cloud-config: ""
# cert-dir: ""
# allow-metric-labels-manifest: ""
# allow-metric-labels: ""
# disabled-metrics: ""
# leader-migration-config: ""
# master: ""
# pv-recycler-pod-template-filepath-hostpath: ""
# pv-recycler-pod-template-filepath-nfs: ""
# service-cluster-ip-range: ""
# show-hidden-metrics-for-version: ""
# tls-cert-file: ""
# tls-cipher-suites: ""
# tls-min-version: ""
# tls-private-key-file: ""
# tls-sni-cert-key: ""
# vmodule: ""
# volume-host-cidr-denylist: ""
# external-cloud-volume-plugin: ""
# requestheader-allowed-names: ""
scheduler:
extraArgs:
authentication-kubeconfig: "/etc/kubernetes/scheduler.conf"
authentication-skip-lookup: "false"
authentication-token-webhook-cache-ttl: "10s"
authentication-tolerate-lookup-failure: "true"
authorization-always-allow-paths: "/healthz,/readyz,/livez,/metrics"
authorization-kubeconfig: "/etc/kubernetes/scheduler.conf"
authorization-webhook-cache-authorized-ttl: "10s"
authorization-webhook-cache-unauthorized-ttl: "10s"
bind-address: "0.0.0.0"
client-ca-file: ""
contention-profiling: "true"
help: "false"
http2-max-streams-per-connection: "0"
kube-api-burst: "100"
kube-api-content-type: "application/vnd.kubernetes.protobuf"
kube-api-qps: "50"
kubeconfig: "/etc/kubernetes/scheduler.conf"
leader-elect: "true"
leader-elect-lease-duration: "15s"
leader-elect-renew-deadline: "10s"
leader-elect-resource-lock: "leases"
leader-elect-resource-name: "kube-scheduler"
leader-elect-resource-namespace: "kube-system"
leader-elect-retry-period: "2s"
log-flush-frequency: "5s"
log-json-info-buffer-size: "0"
log-json-split-stream: "false"
log-text-info-buffer-size: "0"
log-text-split-stream: "false"
logging-format: "text"
permit-address-sharing: "false"
permit-port-sharing: "false"
pod-max-in-unschedulable-pods-duration: "5m0s"
profiling: "true"
requestheader-extra-headers-prefix: "[x-remote-extra-]"
requestheader-group-headers: "[x-remote-group]"
requestheader-username-headers: "[x-remote-user]"
secure-port: "10259"
v: "2"
version: "false"
# allow-metric-labels: "[]"
# allow-metric-labels-manifest: ""
# cert-dir: ""
# config: ""
# disabled-metrics: "[]"
# feature-gates: ""
# master: ""
# requestheader-allowed-names: "[]"
# requestheader-client-ca-file: ""
# show-hidden-metrics-for-version: ""
# tls-cert-file: ""
# tls-cipher-suites: "[]"
# tls-min-version: ""
# tls-private-key-file: ""
# tls-sni-cert-key: "[]"
# vmodule: ""
# write-config-to: ""
Конфигурация Kubeadm
- master-2
- master-3
export HOST_NAME=master-2
export HOST_NAME=master-3
Конфигурация Kubeadm для подключения мастер ноды к кластеру
Переменные окружения для шаблона конфигурационного файла
export MACHINE_LOCAL_ADDRESS="$(ip -4 addr show scope global | awk '/inet/ {print $2; exit}' | cut -d/ -f1)"
export CLUSTER_API_ENDPOINT=api.my-first-cluster.example.com
export CERTIFICATE_UPLOAD_KEY=0c00c2fd5c67c37656c00d78a9d7e1f2eb794ef8e4fc3e2a4b532eb14323cd59
Конфигурационный файл kubeadm для подключения мастера к кластеру
cat <<EOF > /var/run/kubeadm/kubeadm.yaml
---
apiVersion: kubeadm.k8s.io/v1beta3
controlPlane:
localAPIEndpoint:
advertiseAddress: ${MACHINE_LOCAL_ADDRESS}
bindPort: 6443
certificateKey: ${CERTIFICATE_UPLOAD_KEY}
discovery:
bootstrapToken:
apiServerEndpoint: ${CLUSTER_API_ENDPOINT}:6443
unsafeSkipCAVerification: true
token: "fjt9ex.lwzqgdlvoxtqk4yw"
kind: JoinConfiguration
EOF
Проверка настройки
Проверка настройки
ls -al /var/run/kubeadm/kubeadm.yaml
-rw-r--r-- 1 root root 6463 Feb 18 15:20 /var/run/kubeadm/kubeadm.yaml
Настройка Kubernetes Audit
● Не обязателен к применению
Настройка Kubernetes Audit
● Не обязателен к применению
Этапы настройки компонента
- Создание рабочей директории
- Подготовка политики аудита
Создание рабочей директории
mkdir -p /var/log/kubernetes/audit
Подготовка политики аудита
cat <<EOF > /var/log/kubernetes/audit/audit-policy.yaml
---
apiVersion: audit.k8s.io/v1
kind: Policy
# Общие правила
# Исключаем раннюю стадию аудита "RequestReceived", чтобы снизить объем логов и дублирование
# Эта настройка применяется глобально, но в некоторых правилах переопределяется локально
# omitStages может быть указано также внутри отдельных правил
rules:
# Отключаем логирование "watch"-запросов от kube-proxy к endpoint'ам и сервисам
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # Core API group
resources: ["endpoints", "services", "services/status"]
# Отключаем логирование чтения configmap в kube-system от "system:unsecured"
- level: None
users: ["system:unsecured"]
namespaces: ["kube-system"]
verbs: ["get"]
resources:
- group: ""
resources: ["configmaps"]
# Отключаем логирование чтения узлов legacy-пользователем "kubelet"
- level: None
users: ["kubelet"]
verbs: ["get"]
resources:
- group: ""
resources: ["nodes", "nodes/status"]
# Отключаем логирование чтения узлов группой "system:nodes"
- level: None
userGroups: ["system:nodes"]
verbs: ["get"]
resources:
- group: ""
resources: ["nodes", "nodes/status"]
# Отключаем логирование get/update endpoint'ов в kube-system от контроллеров
- level: None
users:
- system:kube-controller-manager
- system:kube-scheduler
- system:serviceaccount:kube-system:endpoint-controller
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: ""
resources: ["endpoints"]
# Отключаем логирование операций с namespace'ами от системного пользователя apiserver
- level: None
users: ["system:apiserver"]
verbs: ["get"]
resources:
- group: ""
resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
# Отключаем логирование операций с configmap и endpoint в kube-system от cluster-autoscaler
- level: None
users: ["cluster-autoscaler"]
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: ""
resources: ["configmaps", "endpoints"]
# Отключаем логирование запросов к метрикам от kube-controller-manager
- level: None
users: ["system:kube-controller-manager"]
verbs: ["get", "list"]
resources:
- group: "metrics.k8s.io"
# Отключаем логирование системных non-resource URL (здоровье, версия, swagger и т.п.)
- level: None
nonResourceURLs:
- /healthz*
- /version
- /swagger*
# Отключаем логирование событий (events) — они часто шумные и не критичны
- level: None
resources:
- group: ""
resources: ["events"]
# Логирование обновлений статуса узлов и подов от kubelet и node-problem-detector
- level: Request
users:
- kubelet
- system:node-problem-detector
- system:serviceaccount:kube-system:node-problem-detector
verbs:
- update
- patch
resources:
- group: ""
resources:
- nodes/status
- pods/status
omitStages:
- "RequestReceived"
# То же самое для всех участников группы system:nodes
- level: Request
userGroups: ["system:nodes"]
verbs:
- update
- patch
resources:
- group: ""
resources:
- nodes/status
- pods/status
omitStages:
- "RequestReceived"
# Логирование массового удаления (deletecollection) от namespace-controller
- level: Request
users: ["system:serviceaccount:kube-system:namespace-controller"]
verbs: ["deletecollection"]
omitStages:
- "RequestReceived"
# Логирование метаданных для чувствительных ресурсов: секретов, токенов, токен-рецензий
- level: Metadata
resources:
- group: ""
resources: ["secrets", "configmaps", "serviceaccounts/token"]
- group: authentication.k8s.io
resources: ["tokenreviews"]
omitStages:
- "RequestReceived"
# Логирование всех безопасных операций чтения (get/list/watch) по всем известным группам
- level: Request
verbs: ["get", "list", "watch"]
resources:
- group: "" # Core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "settings.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Логирование всех операций, включая тело запроса и ответа (RequestResponse)
- level: RequestResponse
resources:
- group: "" # Core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "settings.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Финальный catch-all: логируем метаданные всего остального
- level: Metadata
omitStages:
- "RequestReceived"
EOF