Kubernetes 之APIServer部署
1. 环境准备
IP地址 主机名
10.65.6.3 k8s-master-001
10.65.6.4 k8s-master-002
10.65.6.5 k8s-master-003
K8S API对外域名：noah.api.frank.com.cn
kubernettes Version: 1.18.8
2. 证书签发
2.1 证书环境变量
# touch k8s_env.sh
# 设置证书环境变量
# 设置证书使用时间87600h 10年
export EXPIRY_TIME="87600h"
# kube-apiserver 服务器IP 如果外部访问K8s 集群使用VIP ip 请在下面添加vip ip
export K8S_APISERVER_VIP="\"10.65.6.3\",\"10.65.6.4\",\"10.65.6.5\""
# kubernetes 服务 IP (一般是 SERVICE_CIDR 中第一个IP)
export CLUSTER_KUBERNETES_SVC_IP="10.66.0.1"
# 设置集群参数
export CLUSTER_NAME=kubernetes
export KUBE_API=https://frank.api.baowei-inc.com:5443
# K8S 外部IP 这里高可用使用本地环回IP
export K8S_VIP_DOMAIN=10.65.6.221
export K8S_SSL="\"${K8S_VIP_DOMAIN}\""
#证书所需要的配置参数
export CERT_ST="ShangHai"
export CERT_L="ShangHai"
export CERT_O="k8s"
export CERT_OU="Frank"
export CERT_PROFILE="kubernetes"
# 生成 EncryptionConfig 所需的加密 key
export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
# 设置工作目录
export HOST_PATH=`pwd`
# 设置连接KUBE_APISERVER ip
export KUBE_APISERVER=https://noah.api.baowei-inc.com:5443
#集群域名
export CLUSTER_DNS_DOMAIN="cluster.local"
#source k8s_env.sh
2.2 创建证书
# etcd 如果已经创建可以不用重复创建
# 创建etcd K8S 证书json 存放目录
mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}
# 创建签发证书存放目录
mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}
# CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage，过期时间、服务端认证、客户端认证、加密等)，后续在签名其它证书时需要指定特定场景。
cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json
{
"signing": {
"default": {
"expiry": "${EXPIRY_TIME}"
},
"profiles": {
"${CERT_PROFILE}": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "${EXPIRY_TIME}"
}
}
}
}
EOF
# 创建 Kubernetes CA 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-ca-csr.json
{
"CN": "$CLUSTER_NAME",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
],
"ca": {
"expiry": "${EXPIRY_TIME}"
}
}
EOF
# 生成 Kubernetes CA 证书和私钥
cfssl gencert -initca ${HOST_PATH}/cfssl/k8s/k8s-ca-csr.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-ca
# 创建 Kubernetes API Server 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-apiserver.json
{
"CN": "$CLUSTER_NAME",
"hosts": [
${K8S_APISERVER_VIP},
"${CLUSTER_KUBERNETES_SVC_IP}",
${K8S_SSL},
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.${CLUSTER_DNS_DOMAIN}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 Kubernetes API Server 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/k8s/k8s-apiserver.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-server
# 创建 Kubernetes webhook 证书配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/aggregator.json
{
"CN": "aggregator",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 Kubernetes webhook 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/k8s/aggregator.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/aggregator
# 创建admin管理员 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-apiserver-admin.json
{
"CN": "admin",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "system:masters",
"OU": "Kubernetes-manual"
}
]
}
EOF
# 生成 Kubernetes admin管理员证书
cfssl gencert -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/k8s/k8s-apiserver-admin.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin
2.3 生成的证书到所有需要部署kube-apiserver 节点
#mkdir /apps/conf/kubernetes/ssl/{etcd,k8s} -pv
#scp -r ./cfssl/pki/k8s/* 10.65.6.3:/apps/conf/kubernetes/ssl/k8s
#scp -r ./cfssl/pki/etcd/* 10.65.6.3:/apps/conf/kubernetes/ssl/etcd
2.4 部署kube-apiserver
# 在kubernetes集群节点上部署apiserver
#tar -xvf kubernetes-server-linux-amd64.tar.gz
#mkdir /apps/svr/kubernetes-1.18.8/bin
#cp kubernetes/server/bin/* /apps/svr/kubernetes-1.18.8/bin
#ln -svfn /apps/svr/kubernetes-1.18.8 /apps/svr/kubernetes
2.5 kub-apiserver配置文件准备
# 生成encryption-config.yaml
cat << EOF | tee ${HOST_PATH}/encryption-config.yaml
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: ${ENCRYPTION_KEY}
- identity: {}
EOF
# 创建审计策略文件
cat << EOF | tee ${HOST_PATH}/audit-policy.yaml
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
# The following requests were manually identified as high-volume and low-risk, so drop them.
- level: None
resources:
- group: ""
resources:
- endpoints
- services
- services/status
users:
- 'system:kube-proxy'
verbs:
- watch
- level: None
resources:
- group: ""
resources:
- nodes
- nodes/status
userGroups:
- 'system:nodes'
verbs:
- get
- level: None
namespaces:
- kube-system
resources:
- group: ""
resources:
- endpoints
users:
- 'system:kube-controller-manager'
- 'system:kube-scheduler'
- 'system:serviceaccount:kube-system:endpoint-controller'
verbs:
- get
- update
- level: None
resources:
- group: ""
resources:
- namespaces
- namespaces/status
- namespaces/finalize
users:
- 'system:apiserver'
verbs:
- get
# Don't log HPA fetching metrics.
- level: None
resources:
- group: metrics.k8s.io
users:
- 'system:kube-controller-manager'
verbs:
- get
- list
# Don't log these read-only URLs.
- level: None
nonResourceURLs:
- '/healthz*'
- /version
- '/swagger*'
# Don't log events requests.
- level: None
resources:
- group: ""
resources:
- events
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
users:
- kubelet
- 'system:node-problem-detector'
- 'system:serviceaccount:kube-system:node-problem-detector'
verbs:
- update
- patch
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
userGroups:
- 'system:nodes'
verbs:
- update
- patch
# deletecollection calls can be large, don't log responses for expected namespace deletions
- level: Request
omitStages:
- RequestReceived
users:
- 'system:serviceaccount:kube-system:namespace-controller'
verbs:
- deletecollection
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
# so only log at the Metadata level.
- level: Metadata
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- secrets
- configmaps
- group: authentication.k8s.io
resources:
- tokenreviews
# Get repsonses can be large; skip them.
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
verbs:
- get
- list
- watch
# Default level for known APIs
- level: RequestResponse
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
# Default level for all other requests.
- level: Metadata
omitStages:
- RequestReceived
EOF
##2.7 配置文件分发至 配置目录
#scp -r {audit-policy.yaml,encryption-config.yaml} 10.65.6.3:/apps/conf/kubernetes/
#kube-apiserver配置文件(10.65.6.3)
#cat /apps/conf/kubernetes/kube-apiserver.cfg
KUBE_ETCD_SERVERS="--etcd-servers=https://k8s-master-etcd-001.frank.com.cn:2379,https://k8s-master-etcd-002.frank.com.cn:2379,https://k8s-master-etcd-003.frank.com.cn:2379 "
KUBE_ETCD_CAFILE="--etcd-cafile=/apps/conf/kubernetes/ssl/etcd/etcd-ca.pem"
KUBE_ETCD_CERTFILE="--etcd-certfile=/apps/conf/kubernetes/ssl/etcd/etcd-client.pem"
KUBE_ETCD_KEYFILE="--etcd-keyfile=/apps/conf/kubernetes/ssl/etcd/etcd-client-key.pem"
KUBE_LOGTOSTDERR="--logtostderr=false"
KUBE_BIND_ADDRESS="--bind-address=10.65.6.3"
KUBE_ADVERTISE_ADDRESS="--advertise-address=10.65.6.3"
KUBE_SECURE_PORT="--secure-port=6443"
KUBE_INSECURE_PORT="--insecure-port=0"
KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.66.0.0/16"
KUBE_SERVICE_NODE_PORT_RANGE="--service-node-port-range=30000-65535"
KUBE_CLIENT_CA_FILE="--client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_TLS_CERT_FILE="--tls-cert-file=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_KUBELET_CLIENT_CERTIFICATE="--kubelet-client-certificate=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_KUBELET_CLIENT_KEY="--kubelet-client-key=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_SERVICE_ACCOUNT_KEY_FILE="--service-account-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_REQUESTHEADER_CLIENT_CA_FILE="--requestheader-client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_PROXY_CLIENT_CERT_FILE="--proxy-client-cert-file=/apps/conf/kubernetes/ssl/k8s/aggregator.pem"
KUBE_PROXY_CLIENT_KEY_FILE="--proxy-client-key-file=/apps/conf/kubernetes/ssl/k8s/aggregator-key.pem"
KUBE_REQUESTHEADER_ALLOWED_NAMES="--requestheader-allowed-names=aggregator"
KUBE_REQUESTHEADER_GROUP_HEADERS="--requestheader-group-headers=X-Remote-Group"
KUBE_REQUESTHEADER_EXTRA_HEADERS_PREFIX="--requestheader-extra-headers-prefix=X-Remote-Extra-"
KUBE_REQUESTHEADER_USERNAME_HEADERS="--requestheader-username-headers=X-Remote-User"
KUBE_ENABLE_AGGREGATOR_ROUTING="--enable-aggregator-routing=true"
KUBE_ANONYMOUS_AUTH="--anonymous-auth=false"
KUBE_ENCRYPTION_PROVIDER_CONFIG="--encryption-provider-config=/apps/conf/kubernetes/encryption-config.yaml"
KUBE_ENABLE_ADMISSION_PLUGINS="--enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook"
KUBE_DISABLE_ADMISSION_PLUGINS="--disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy"
KUBE_CORS_ALLOWED_ORIGINS="--cors-allowed-origins=.*"
KUBE_RUNTIME_CONFIG="--runtime-config=api/all=true"
KUBE_KUBELET_PREFERRED_ADDRESS_TYPES="--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname"
KUBE_AUTHORIZATION_MODE="--authorization-mode=Node,RBAC"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
KUBE_APISERVER_COUNT="--apiserver-count=1"
KUBE_AUDIT_DYNAMIC_CONFIGURATION="--audit-dynamic-configuration"
KUBE_AUDIT_LOG_MAXAGE="--audit-log-maxage=30"
KUBE_AUDIT_LOG_MAXBACKUP="--audit-log-maxbackup=3"
KUBE_AUDIT_LOG_MAXSIZE="--audit-log-maxsize=100"
KUBE_AUDIT_LOG_TRUNCATE_ENABLED="--audit-log-truncate-enabled"
KUBE_AUDIT_POLICY_FILE="--audit-policy-file=/apps/conf/kubernetes/audit-policy.yaml"
KUBE_AUDIT_LOG_PATH="--audit-log-path=/apps/logs/kubernetes/api-server-audit.log"
KUBE_PROFILING="--profiling=true"
KUBE_KUBELET_HTTPS="--kubelet-https=true"
KUBE_EVENT_TTL="--event-ttl=1h"
KUBE_FEATURE_GATES="--feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true"
KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH="--enable-bootstrap-token-auth=true"
KUBE_TOKEN_AUTH_FILE="--token-auth-file=/apps/conf/kubernetes/token.csv"
KUBE_ALSOLOGTOSTDERR="--alsologtostderr=true"
KUBE_LOG_DIR="--log-dir=/apps/logs/kubernetes"
KUBE_LOG_FILE_MAX_SIZE="--log-file-max-size=1024"
KUBE_LOG_LEVEL="--v=2"
KUBE_ENDPOINT_RECONCILER_TYPE="--endpoint-reconciler-type=lease"
KUBE_MAX_MUTATING_REQUESTS_INFLIGHT="--max-mutating-requests-inflight=500"
KUBE_MAX_REQUESTS_INFLIGHT="--max-requests-inflight=2000"
KUBE_MIN_REQUEST_TIMEOUT="--min-request-timeout=3600"
KUBE_TARGET_RAM_MB="--target-ram-mb=600"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
#kube-apiserver配置10.65.6.4
cat /apps/conf/kubernetes/kube-apiserver.cfg
KUBE_ETCD_SERVERS="--etcd-servers=https://k8s-master-etcd-001.frank.com.cn:2379,https://k8s-master-etcd-002.frank.com.cn:2379,https://k8s-master-etcd-003.frank.com.cn:2379 "
KUBE_ETCD_CAFILE="--etcd-cafile=/apps/conf/kubernetes/ssl/etcd/etcd-ca.pem"
KUBE_ETCD_CERTFILE="--etcd-certfile=/apps/conf/kubernetes/ssl/etcd/etcd-client.pem"
KUBE_ETCD_KEYFILE="--etcd-keyfile=/apps/conf/kubernetes/ssl/etcd/etcd-client-key.pem"
KUBE_LOGTOSTDERR="--logtostderr=false"
KUBE_BIND_ADDRESS="--bind-address=10.65.6.4"
KUBE_ADVERTISE_ADDRESS="--advertise-address=10.65.6.4"
KUBE_SECURE_PORT="--secure-port=6443"
KUBE_INSECURE_PORT="--insecure-port=0"
KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.66.0.0/16"
KUBE_SERVICE_NODE_PORT_RANGE="--service-node-port-range=30000-65535"
KUBE_CLIENT_CA_FILE="--client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_TLS_CERT_FILE="--tls-cert-file=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_KUBELET_CLIENT_CERTIFICATE="--kubelet-client-certificate=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_KUBELET_CLIENT_KEY="--kubelet-client-key=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_SERVICE_ACCOUNT_KEY_FILE="--service-account-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_REQUESTHEADER_CLIENT_CA_FILE="--requestheader-client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_PROXY_CLIENT_CERT_FILE="--proxy-client-cert-file=/apps/conf/kubernetes/ssl/k8s/aggregator.pem"
KUBE_PROXY_CLIENT_KEY_FILE="--proxy-client-key-file=/apps/conf/kubernetes/ssl/k8s/aggregator-key.pem"
KUBE_REQUESTHEADER_ALLOWED_NAMES="--requestheader-allowed-names=aggregator"
KUBE_REQUESTHEADER_GROUP_HEADERS="--requestheader-group-headers=X-Remote-Group"
KUBE_REQUESTHEADER_EXTRA_HEADERS_PREFIX="--requestheader-extra-headers-prefix=X-Remote-Extra-"
KUBE_REQUESTHEADER_USERNAME_HEADERS="--requestheader-username-headers=X-Remote-User"
KUBE_ENABLE_AGGREGATOR_ROUTING="--enable-aggregator-routing=true"
KUBE_ANONYMOUS_AUTH="--anonymous-auth=false"
KUBE_ENCRYPTION_PROVIDER_CONFIG="--encryption-provider-config=/apps/conf/kubernetes/encryption-config.yaml"
KUBE_ENABLE_ADMISSION_PLUGINS="--enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook"
KUBE_DISABLE_ADMISSION_PLUGINS="--disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy"
KUBE_CORS_ALLOWED_ORIGINS="--cors-allowed-origins=.*"
KUBE_RUNTIME_CONFIG="--runtime-config=api/all=true"
KUBE_KUBELET_PREFERRED_ADDRESS_TYPES="--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname"
KUBE_AUTHORIZATION_MODE="--authorization-mode=Node,RBAC"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
KUBE_APISERVER_COUNT="--apiserver-count=1"
KUBE_AUDIT_DYNAMIC_CONFIGURATION="--audit-dynamic-configuration"
KUBE_AUDIT_LOG_MAXAGE="--audit-log-maxage=30"
KUBE_AUDIT_LOG_MAXBACKUP="--audit-log-maxbackup=3"
KUBE_AUDIT_LOG_MAXSIZE="--audit-log-maxsize=100"
KUBE_AUDIT_LOG_TRUNCATE_ENABLED="--audit-log-truncate-enabled"
KUBE_AUDIT_POLICY_FILE="--audit-policy-file=/apps/conf/kubernetes/audit-policy.yaml"
KUBE_AUDIT_LOG_PATH="--audit-log-path=/apps/logs/kubernetes/api-server-audit.log"
KUBE_PROFILING="--profiling=true"
KUBE_KUBELET_HTTPS="--kubelet-https=true"
KUBE_EVENT_TTL="--event-ttl=1h"
KUBE_FEATURE_GATES="--feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true"
KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH="--enable-bootstrap-token-auth=true"
KUBE_TOKEN_AUTH_FILE="--token-auth-file=/apps/conf/kubernetes/token.csv"
KUBE_ALSOLOGTOSTDERR="--alsologtostderr=true"
KUBE_LOG_DIR="--log-dir=/apps/logs/kubernetes"
KUBE_LOG_FILE_MAX_SIZE="--log-file-max-size=1024"
KUBE_LOG_LEVEL="--v=2"
KUBE_ENDPOINT_RECONCILER_TYPE="--endpoint-reconciler-type=lease"
KUBE_MAX_MUTATING_REQUESTS_INFLIGHT="--max-mutating-requests-inflight=500"
KUBE_MAX_REQUESTS_INFLIGHT="--max-requests-inflight=2000"
KUBE_MIN_REQUEST_TIMEOUT="--min-request-timeout=3600"
KUBE_TARGET_RAM_MB="--target-ram-mb=600"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
#kube-apiserver 配置10.65.6.5
KUBE_ETCD_SERVERS="--etcd-servers=https://k8s-master-etcd-001.frank.com.cn:2379,https://k8s-master-etcd-002.frank.com.cn:2379,https://k8s-master-etcd-003.frank.com.cn:2379 "
KUBE_ETCD_CAFILE="--etcd-cafile=/apps/conf/kubernetes/ssl/etcd/etcd-ca.pem"
KUBE_ETCD_CERTFILE="--etcd-certfile=/apps/conf/kubernetes/ssl/etcd/etcd-client.pem"
KUBE_ETCD_KEYFILE="--etcd-keyfile=/apps/conf/kubernetes/ssl/etcd/etcd-client-key.pem"
KUBE_LOGTOSTDERR="--logtostderr=false"
KUBE_BIND_ADDRESS="--bind-address=10.65.6.5"
KUBE_ADVERTISE_ADDRESS="--advertise-address=10.65.6.5"
KUBE_SECURE_PORT="--secure-port=6443"
KUBE_INSECURE_PORT="--insecure-port=0"
KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.66.0.0/16"
KUBE_SERVICE_NODE_PORT_RANGE="--service-node-port-range=30000-65535"
KUBE_CLIENT_CA_FILE="--client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_TLS_CERT_FILE="--tls-cert-file=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_KUBELET_CLIENT_CERTIFICATE="--kubelet-client-certificate=/apps/conf/kubernetes/ssl/k8s/k8s-server.pem"
KUBE_KUBELET_CLIENT_KEY="--kubelet-client-key=/apps/conf/kubernetes/ssl/k8s/k8s-server-key.pem"
KUBE_SERVICE_ACCOUNT_KEY_FILE="--service-account-key-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_REQUESTHEADER_CLIENT_CA_FILE="--requestheader-client-ca-file=/apps/conf/kubernetes/ssl/k8s/k8s-ca.pem"
KUBE_PROXY_CLIENT_CERT_FILE="--proxy-client-cert-file=/apps/conf/kubernetes/ssl/k8s/aggregator.pem"
KUBE_PROXY_CLIENT_KEY_FILE="--proxy-client-key-file=/apps/conf/kubernetes/ssl/k8s/aggregator-key.pem"
KUBE_REQUESTHEADER_ALLOWED_NAMES="--requestheader-allowed-names=aggregator"
KUBE_REQUESTHEADER_GROUP_HEADERS="--requestheader-group-headers=X-Remote-Group"
KUBE_REQUESTHEADER_EXTRA_HEADERS_PREFIX="--requestheader-extra-headers-prefix=X-Remote-Extra-"
KUBE_REQUESTHEADER_USERNAME_HEADERS="--requestheader-username-headers=X-Remote-User"
KUBE_ENABLE_AGGREGATOR_ROUTING="--enable-aggregator-routing=true"
KUBE_ANONYMOUS_AUTH="--anonymous-auth=false"
KUBE_ENCRYPTION_PROVIDER_CONFIG="--encryption-provider-config=/apps/conf/kubernetes/encryption-config.yaml"
KUBE_ENABLE_ADMISSION_PLUGINS="--enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook"
KUBE_DISABLE_ADMISSION_PLUGINS="--disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy"
KUBE_CORS_ALLOWED_ORIGINS="--cors-allowed-origins=.*"
KUBE_RUNTIME_CONFIG="--runtime-config=api/all=true"
KUBE_KUBELET_PREFERRED_ADDRESS_TYPES="--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname"
KUBE_AUTHORIZATION_MODE="--authorization-mode=Node,RBAC"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
KUBE_APISERVER_COUNT="--apiserver-count=1"
KUBE_AUDIT_DYNAMIC_CONFIGURATION="--audit-dynamic-configuration"
KUBE_AUDIT_LOG_MAXAGE="--audit-log-maxage=30"
KUBE_AUDIT_LOG_MAXBACKUP="--audit-log-maxbackup=3"
KUBE_AUDIT_LOG_MAXSIZE="--audit-log-maxsize=100"
KUBE_AUDIT_LOG_TRUNCATE_ENABLED="--audit-log-truncate-enabled"
KUBE_AUDIT_POLICY_FILE="--audit-policy-file=/apps/conf/kubernetes/audit-policy.yaml"
KUBE_AUDIT_LOG_PATH="--audit-log-path=/apps/logs/kubernetes/api-server-audit.log"
KUBE_PROFILING="--profiling=true"
KUBE_KUBELET_HTTPS="--kubelet-https=true"
KUBE_EVENT_TTL="--event-ttl=1h"
KUBE_FEATURE_GATES="--feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true"
KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH="--enable-bootstrap-token-auth=true"
KUBE_TOKEN_AUTH_FILE="--token-auth-file=/apps/conf/kubernetes/token.csv"
KUBE_ALSOLOGTOSTDERR="--alsologtostderr=true"
KUBE_LOG_DIR="--log-dir=/apps/logs/kubernetes"
KUBE_LOG_FILE_MAX_SIZE="--log-file-max-size=1024"
KUBE_LOG_LEVEL="--v=2"
KUBE_ENDPOINT_RECONCILER_TYPE="--endpoint-reconciler-type=lease"
KUBE_MAX_MUTATING_REQUESTS_INFLIGHT="--max-mutating-requests-inflight=500"
KUBE_MAX_REQUESTS_INFLIGHT="--max-requests-inflight=2000"
KUBE_MIN_REQUEST_TIMEOUT="--min-request-timeout=3600"
KUBE_TARGET_RAM_MB="--target-ram-mb=600"
KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
2.8 调整kube-apiserver systemd文件
cat /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes ApiServer
Documentation=https://github.com/kubernetes/kubernetes
[Service]
Type=notify
EnvironmentFile=/apps/conf/kubernetes/kube-apiserver.cfg
ExecStart=/apps/svr/kubernetes/bin/kube-apiserver \
${KUBE_ETCD_SERVERS} \
${KUBE_ETCD_CAFILE} \
${KUBE_ETCD_KEYFILE} \
${KUBE_ETCD_CERTFILE} \
${KUBE_LOGTOSTDERR} \
${KUBE_BIND_ADDRESS} \
${KUBE_ADVERTISE_ADDRESS} \
${KUBE_SECURE_PORT} \
${KUBE_INSECURE_PORT} \
${KUBE_SERVICE_CLUSTER_IP_RANGE} \
${KUBE_SERVICE_NODE_PORT_RANGE} \
${KUBE_CLIENT_CA_FILE} \
${KUBE_TLS_CERT_FILE} \
${KUBE_TLS_PRIVATE_KEY_FILE} \
${KUBE_KUBELET_CLIENT_CERTIFICATE} \
${KUBE_KUBELET_CLIENT_KEY} \
${KUBE_SERVICE_ACCOUNT_KEY_FILE} \
${KUBE_REQUESTHEADER_CLIENT_CA_FILE} \
${KUBE_PROXY_CLIENT_CERT_FILE} \
${KUBE_PROXY_CLIENT_KEY_FILE} \
${KUBE_REQUESTHEADER_ALLOWED_NAMES} \
${KUBE_REQUESTHEADER_GROUP_HEADERS} \
${KUBE_REQUESTHEADER_EXTRA_HEADERS_PREFIX} \
${KUBE_REQUESTHEADER_USERNAME_HEADERS} \
${KUBE_ENABLE_AGGREGATOR_ROUTING} \
${KUBE_ANONYMOUS_AUTH} \
${KUBE_ENCRYPTION_PROVIDER_CONFIG} \
${KUBE_ENABLE_ADMISSION_PLUGINS} \
${KUBE_DISABLE_ADMISSION_PLUGINS} \
${KUBE_CORS_ALLOWED_ORIGINS} \
${KUBE_RUNTIME_CONFIG} \
${KUBE_KUBELET_PREFERRED_ADDRESS_TYPES} \
${KUBE_AUTHORIZATION_MODE} \
${KUBE_ALLOW_PRIVILEGED} \
${KUBE_APISERVER_COUNT} \
${KUBE_AUDIT_DYNAMIC_CONFIGURATION} \
${KUBE_AUDIT_LOG_MAXAGE} \
${KUBE_AUDIT_LOG_MAXBACKUP} \
${KUBE_AUDIT_LOG_MAXSIZE} \
${KUBE_AUDIT_LOG_TRUNCATE_ENABLED} \
${KUBE_AUDIT_POLICY_FILE} \
${KUBE_AUDIT_LOG_PATH} \
${KUBE_PROFILING} \
${KUBE_KUBELET_HTTPS} \
${KUBE_EVENT_TTL} \
${KUBE_FEATURE_GATES} \
${KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH} \
${KUBE_TOKEN_AUTH_FILE} \
${KUBE_ALSOLOGTOSTDERR} \
${KUBE_LOG_DIR} \
${KUBE_LOG_FILE_MAX_SIZE} \
${KUBE_LOG_LEVEL} \
${KUBE_ENDPOINT_RECONCILER_TYPE} \
${KUBE_MAX_MUTATING_REQUESTS_INFLIGHT} \
${KUBE_MAX_REQUESTS_INFLIGHT} \
${KUBE_MIN_REQUEST_TIMEOUT} \
${KUBE_TARGET_RAM_MB} \
${KUBE_ALLOW_PRIVILEGED} \
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
2.9 创建 TLS Bootstrapping Token
#BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
BOOTSTRAP_TOKEN=0fb61c46f8991b718eb38d27b605b008
cat > /apps/conf/kubernetes/token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
2.10 获取token信息
#cat /apps/conf/kubernetes/token.csv
0fb61c46f8991b718eb38d27b605b008,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
2.11 配置文件修改为tokenID
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=b639a2717c79839fd7fab7bac97dca32 \
--kubeconfig=bootstrap.kubeconfig
2.12 创建bootstrap角色赋予权限用于连接apiserver请求签名（关键）
#kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
2.13 启动kube-apiserver 服务
#创建VOLUME
#mkdir /apps/conf/kubernetes/kubelet-plugins/volume -pv
#启动KUBE-APISERVER,并设置开启启动
#systemctl daemon-reload
#systemctl start kube-apiserver.service
#systemctl enable kube-apiserver.service
2.14 验证服务状态
#创建 kubeconfig 文件夹
mkdir -p ${HOST_PATH}/kubeconfig
# 创建admin管理员登录kubeconfig
# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
--certificate-authority=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
--embed-certs=true \
--server=${KUBE_API} \
--kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials admin \
--client-certificate=${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin.pem \
--client-key=${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin-key.pem \
--embed-certs=true \
--kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 设置上下文参数
kubectl config set-context ${CLUSTER_NAME} \
--cluster=${CLUSTER_NAME} \
--user=admin \
--namespace=kube-system \
--kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 设置默认上下文
kubectl config use-context ${CLUSTER_NAME} --kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 创建当前家目录.kube 目录
mkdir -p ~/.kube
cp ${HOST_PATH}/kubeconfig/admin.kubeconfig ~/.kube/config
# 验证集群
kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
etcd-1 Healthy {"health":"true"}
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
#kubectl cluster-info
Kubernetes master is running at https://bw.api.baowei-inc.com:6443
作者:Frank_弗兰克
原文链接:https://www.jianshu.com/p/78250cda9b2e