k8s-安装文档-01-master节点基础-二进制方式部署文档（线上版本）

2023-06-01 00:00:00 K8S

K8S二进制方式部署文档（线上版本）

## 说明及参考

>二进制版本发布信息官方

https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG.md

## 初始化环境

hostnamectl --static set-hostname hostname
k8s-m-157 10.0.1.157
k8s-m-158 10.0.1.158
k8s-m-159 10.0.1.159

#编辑 /etc/hosts 文件，配置hostname 通信

vi /etc/hosts
10.0.1.157 k8s-m-157
10.0.1.158 k8s-m-158
10.0.1.159 k8s-m-159

## 安装 cfssl（创建验证证书工具）

mkdir -p /opt/local/cfssl
cd /opt/local/cfssl
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
mv cfssl_linux-amd64 cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
mv cfssljson_linux-amd64 cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 cfssl-certinfo
chmod +x *

# 安装docker（17-03CE）

yum localinstall https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm
yum localinstall https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm

## docker service的配置文件如下：vi /usr/lib/systemd/system/docker.service

[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target firewalld.service
[Service]
EnvironmentFile=-/run/flannel/docker
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
Environment=HTTP_PROXY=http://10.0.1.110:1080/
Environment=HTTPS_PROXY=http://10.0.1.110:1080/
Environment=NO_PROXY=localhost,127.0.0.1,5zrvze64.mirror.aliyuncs.com,docker.io,registry.cn-hangzhou.aliyuncs.com,10.0.1.147
ExecStart=/usr/bin/dockerd --bip=172.17.0.1/16 --registry-mirror=https://5zrvze64.mirror.aliyuncs.com  --insecure-registry=10.0.1.147  --log-driver=json-file     --log-opt max-size=100m --log-opt max-file=10
ExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
#TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

## 重新读取配置，启动 docker

systemctl daemon-reload
systemctl start docker
systemctl enable docker

# etcd 集群

> etcd 是k8s集群的基础组件

## 安装 etcd

yum -y install etcd

## 创建 CA 证书配置

mkdir /opt/ssl
cd /opt/ssl

# config.json 文件

vi  config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}

# csr.json 文件

vi csr.json
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

## 生成 CA 证书和私钥

cd /opt/ssl/
/opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca

## 分发证书

# 创建证书目录

mkdir -p /etc/kubernetes/ssl

# 拷贝所有文件到目录下

cp *.pem /etc/kubernetes/ssl
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 创建 etcd 证书

cd /opt/ssl/
vi etcd-csr.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "10.0.1.157",
    "10.0.1.158",
    "10.0.1.159"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

# 生成 etcd 密钥

/opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \
  -ca-key=/opt/ssl/ca-key.pem \
  -config=/opt/ssl/config.json \
  -profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd

# 查看生成

# ls etcd*
etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem

# 拷贝到kubernetes目录

cp etcd*.pem /etc/kubernetes/ssl/
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## etcd conf配置文件

> 样例为157的。158、159类似

[[email protected] ~]# cat /etc/etcd/etcd.conf
# [member]
ETCD_NAME=etcd1
ETCD_DATA_DIR="/var/lib/etcd/etcd1.etcd"
ETCD_WAL_DIR="/var/lib/etcd/wal"
ETCD_SNAPSHOT_COUNT="100"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://10.0.1.157:2380"
ETCD_LISTEN_CLIENT_URLS="https://10.0.1.157:2379,http://127.0.0.1:2379"
ETCD_MAX_SNAPSHOTS="5"
ETCD_MAX_WALS="5"
#ETCD_CORS=""
# [cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.1.157:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd1=https://10.0.1.157:2380,etcd2=https://10.0.1.158:2380,etcd3=https://10.0.1.159:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://10.0.1.157:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_SRV=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_STRICT_RECONFIG_CHECK="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
# [proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
# [security]
ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_AUTO_TLS="true"
# [logging]
#ETCD_DEBUG="false"
# examples for -log-package-levels etcdserver=WARNING,security=DEBUG
#ETCD_LOG_PACKAGE_LEVELS=""

chmod 644 /etc/kubernetes/ssl/etcd-key.pem

## etcd service配置文件

> 样例为157的。158、159类似

[[email protected] ~]# cat /lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

## 启动 etcd

> 分别启动所有节点的 etcd 服务

systemctl enable etcd
systemctl start etcd
systemctl status etcd

# 如果报错请使用

journalctl -f -t etcd  和 journalctl -u etcd 来定位问题

## 验证 etcd 集群状态

> 查看 etcd 集群状态：

etcdctl --endpoints=https://10.0.1.157:2379 \
        --cert-file=/etc/kubernetes/ssl/etcd.pem \
        --ca-file=/etc/kubernetes/ssl/ca.pem \
        --key-file=/etc/kubernetes/ssl/etcd-key.pem \
        cluster-health
#
member 5c0574e3583bc6e2 is healthy: got healthy result from https://10.0.1.158:2379
member 95ea7bb2b81fdd17 is healthy: got healthy result from https://10.0.1.157:2379
member f9fa443b8c78ee60 is healthy: got healthy result from https://10.0.1.159:2379

> 查看 etcd 集群成员：

etcdctl --endpoints=https://10.0.1.157:2379 \
        --cert-file=/etc/kubernetes/ssl/etcd.pem \
        --ca-file=/etc/kubernetes/ssl/ca.pem \
        --key-file=/etc/kubernetes/ssl/etcd-key.pem \
        member list
#
5c0574e3583bc6e2: name=etcd2 peerURLs=https://10.0.1.158:2380 clientURLs=https://10.0.1.158:2379 isLeader=false
95ea7bb2b81fdd17: name=etcd1 peerURLs=https://10.0.1.157:2380 clientURLs=https://10.0.1.157:2379 isLeader=true
f9fa443b8c78ee60: name=etcd3 peerURLs=https://10.0.1.159:2380 clientURLs=https://10.0.1.159:2379 isLeader=false

# K8S

> 下载地址： https://dl.k8s.io/v1.8.6/kubernetes-server-linux-amd64.tar.gz

## 二进制可执行程序安装

#解压找到如下可执行文件

cd /opt/soft
cp kube-apiserver kube-controller-manager kubectl kubelet kube-proxy kube-scheduler /usr/local/bin/
chmod +x  /usr/local/bin/kube*

# 安装 kubectl 工具

## 创建 admin 证书

> kubectl 与 kube-apiserver 的安全端口通信，需要为安全通信提供 TLS 证书和秘钥。

cd /opt/ssl/
vi admin-csr.json
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}

# 生成 admin 证书和私钥

cd /opt/ssl/
/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
  -ca-key=/etc/kubernetes/ssl/ca-key.pem \
  -config=/opt/ssl/config.json \
  -profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin

#分发证书

cp admin*.pem /etc/kubernetes/ssl/
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 配置 kubectl kubeconfig 文件

> server 配置为本机IP 各自连接本机的 Api

# 配置 kubernetes 集群

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=https://127.0.0.1:6443

# 配置客户端认证

kubectl config set-credentials admin \
  --client-certificate=/etc/kubernetes/ssl/admin.pem \
  --embed-certs=true \
  --client-key=/etc/kubernetes/ssl/admin-key.pem
  
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=admin
  
kubectl config use-context kubernetes

rsync -a ~/.kube/ 172.18.1.68:~/.kube/
rsync -a ~/.kube/ 172.18.1.69:~/.kube/

## kubectl config 文件

# kubeconfig 文件在如下:

/root/.kube

# 部署 Kubernetes Master 节点

> Master 需要部署 kube-apiserver , kube-scheduler , kube-controller-manager 这三个组件。

kube-scheduler 作用是调度pods分配到那个node里，简单来说就是资源调度。

kube-controller-manager 作用是对 deployment controller , replication controller, endpoints controller, namespace controller, and serviceaccounts controller等等的循环控制，与kube-apiserver交互。

## 创建 kubernetes 证书

cd /opt/ssl
vi kubernetes-csr.json
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.0.1.157",
    "10.0.1.158",
    "10.0.1.159",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

## 这里 hosts 字段中三个 IP 分别为 127.0.0.1 ,Master 的IP，多个Master需要写多个 10.254.0.1 为 kubernetes SVC 的 IP，一般是部署网络的第一个IP , 如: 10.254.0.1 ，在启动完成后，我们使用 kubectl get svc ，就可以查看到

## 生成 kubernetes 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
  -ca-key=/etc/kubernetes/ssl/ca-key.pem \
  -config=/opt/ssl/config.json \
  -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes

# 拷贝到目录

cp -r kubernetes*.pem /etc/kubernetes/ssl/
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 配置 kube-apiserver

> kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致，如果一致则自动为 kubelet生成证书和秘钥。

# 生成 token

# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
6435c5a809c5d82c978

# 创建 token.csv 文件

cd /opt/ssl
vi token.csv
6435c5a809c5d82c978,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

# 创建 user.csv 文件(baseuser)

vi user.csv
infun123456,infun,1

# 拷贝

cp token.csv /etc/kubernetes/
cp user.csv /etc/kubernetes/

# 生成高级审核配置文件

cd /etc/kubernetes
cat >> audit-policy.yaml <<EOF
# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
- level: Metadata
EOF

rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 创建 kube-apiserver.service 文件

> 样例为157配置、158、159修改ip即可

[root@k8s-m-157 soft]# cat /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
User=root
ExecStart=/usr/local/bin/kube-apiserver \
  --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
  --advertise-address=10.0.1.157 \
  --allow-privileged=true \
  --apiserver-count=3 \
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \
  --audit-log-maxage=30 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-path=/var/log/kubernetes/audit.log \
  --authorization-mode=Node,RBAC \
  --bind-address=0.0.0.0 \
  --secure-port=6443 \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --enable-swagger-ui=true \
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \
  --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \
  --etcd-servers=https://10.0.1.157:2379,https://10.0.1.158:2379,https://10.0.1.159:2379 \
  --event-ttl=1h \
  --kubelet-https=true \
  --insecure-bind-address=127.0.0.1 \
  --basic-auth-file=/etc/kubernetes/user.csv \
  --anonymous-auth=false \
  --insecure-port=8080 \
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --service-cluster-ip-range=10.254.0.0/16 \
  --service-node-port-range=30000-32000 \
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
  --enable-bootstrap-token-auth \
  --token-auth-file=/etc/kubernetes/token.csv \
  --v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

## 启动 kube-apiserver

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

# 部署 Kubernetes Master 节点

> Master 需要部署 kube-apiserver , kube-scheduler , kube-controller-manager 这三个组件。

kube-scheduler 作用是调度pods分配到那个node里，简单来说就是资源调度。

## 创建 kubernetes 证书

cd /opt/ssl
vi kubernetes-csr.json
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.0.1.157",
    "10.0.1.158",
    "10.0.1.159",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

## 生成 kubernetes 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
  -ca-key=/etc/kubernetes/ssl/ca-key.pem \
  -config=/opt/ssl/config.json \
  -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes

# 拷贝到目录

cp -r kubernetes*.pem /etc/kubernetes/ssl/
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 配置 kube-apiserver

# 生成 token

# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
6435c5a809c5d82c978

# 创建 token.csv 文件

cd /opt/ssl
vi token.csv
6435c5a809c5d82c978,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

# 创建 user.csv 文件(baseuser)

vi user.csv
infun123456,infun,1

# 拷贝

cp token.csv /etc/kubernetes/
cp user.csv /etc/kubernetes/

# 生成高级审核配置文件

cd /etc/kubernetes
cat >> audit-policy.yaml <<EOF
# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
- level: Metadata
EOF

rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 创建 kube-apiserver.service 文件

> 样例为157配置、158、159修改ip即可

[root@k8s-m-157 soft]# cat /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
User=root
ExecStart=/usr/local/bin/kube-apiserver \
  --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
  --advertise-address=10.0.1.157 \
  --allow-privileged=true \
  --apiserver-count=3 \
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \
  --audit-log-maxage=30 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-path=/var/log/kubernetes/audit.log \
  --authorization-mode=Node,RBAC \
  --bind-address=0.0.0.0 \
  --secure-port=6443 \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --enable-swagger-ui=true \
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \
  --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \
  --etcd-servers=https://10.0.1.157:2379,https://10.0.1.158:2379,https://10.0.1.159:2379 \
  --event-ttl=1h \
  --kubelet-https=true \
  --insecure-bind-address=127.0.0.1 \
  --basic-auth-file=/etc/kubernetes/user.csv \
  --anonymous-auth=false \
  --insecure-port=8080 \
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --service-cluster-ip-range=10.254.0.0/16 \
  --service-node-port-range=30000-32000 \
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
  --enable-bootstrap-token-auth \
  --token-auth-file=/etc/kubernetes/token.csv \
  --v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

## 启动 kube-apiserver

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

## 配置 kube-controller-manager

> master 配置为各自本地 IP

# 创建 kube-controller-manager.service 文件

[[email protected] soft]# cat /etc/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
  --address=0.0.0.0 \
  --master=http://127.0.0.1:8080 \
  --allocate-node-cidrs=true \
  --service-cluster-ip-range=10.254.0.0/16 \
  --cluster-cidr=10.233.0.0/16 \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
  --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --root-ca-file=/etc/kubernetes/ssl/ca.pem \
  --leader-elect=true \
  --v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target

## 启动 kube-controller-manager

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager

## 配置 kube-scheduler

> master 配置为各自本地 IP

# 创建 kube-cheduler.service 文件

[root@k8s-m-157 soft]# cat /etc/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-scheduler \
  --address=0.0.0.0 \
  --master=http://127.0.0.1:8080 \
  --leader-elect=true \
  --v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target

## 启动 kube-scheduler

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler

## 验证 Master 节点

[[email protected] soft]# kubectl get componentstatuses
NAME                 STATUS    MESSAGE              ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   {"health": "true"}
etcd-2               Healthy   {"health": "true"}
etcd-1               Healthy   {"health": "true"}
[[email protected] ~]# kubectl get componentstatuses
NAME                 STATUS    MESSAGE              ERROR
etcd-2               Healthy   {"health": "true"}
etcd-1               Healthy   {"health": "true"}
etcd-0               Healthy   {"health": "true"}
controller-manager   Healthy   ok
scheduler            Healthy   ok
[[email protected] ~]# kubectl get componentstatuses
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-2               Healthy   {"health": "true"}
etcd-1               Healthy   {"health": "true"}
etcd-0               Healthy   {"health": "true"}

## 部署 Master Node 部分

> Node 部分需要部署的组件有 kubelet kube-proxy 组件。

## 配置 kubelet

> kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper 角色，然后 kubelet 才有权限创建认证请求(certificatesigningrequests)。

# 先创建认证请求

# user 为 master 中 token.csv 文件里配置的用户

# 只需创建一次就可以

kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

## 创建 kubelet kubeconfig 文件

> server 配置为 master 本机 IP

# 配置集群

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=https://127.0.0.1:6443 \
  --kubeconfig=bootstrap.kubeconfig

# 配置客户端认证

kubectl config set-credentials kubelet-bootstrap \
  --token=2c431716c96b85cb96d2f11f1ece1d6f \
  --kubeconfig=bootstrap.kubeconfig

# 配置关联

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig

# 配置默认关联

kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

# 拷贝生成的 bootstrap.kubeconfig 文件

mv bootstrap.kubeconfig /etc/kubernetes/

## 创建 kubelet.service 文件

> 配置为 node 本机 IP 样例159，其它类似

[[email protected] ~]#  cat /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
  --cgroup-driver=cgroupfs \
  --hostname-override=k8s-m-159 \
  --pod-infra-container-image=10.0.1.147/k8s/pause-amd64:3.0 \
  --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
  --cert-dir=/etc/kubernetes/ssl \
  --cluster_dns=10.254.0.2 \
  --cluster_domain=cluster.local. \
  --hairpin-mode promiscuous-bridge \
  --allow-privileged=true \
  --fail-swap-on=false \
  --serialize-image-pulls=false \
  --logtostderr=true \
  --max-pods=512 \
  --network-plugin=cni \
# --network-plugin=cni 先不加这行!
  --v=2
[Install]
WantedBy=multi-user.target

## 启动 kubelet

mkdir /var/lib/kubelet
systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet
systemctl status kubelet

# 如果报错请使用

journalctl -f -t kubelet  和 journalctl -u kubelet 来定位问题

## 配置 TLS 认证

# 查看 csr 的名称

[[email protected] ~]# kubectl get csr
NAME                                                   AGE       REQUESTOR           CONDITION
node-csr-pf-Bb5Iqx6ccvVA67gLVT-G4Zl3Zl5FPUZS4d7V6rk4   1h        kubelet-bootstrap   Pending

# 增加认证,这里先把其它几个节点也加进来csr,再往下面走

[[email protected] ~]# kubectl certificate approve node-csr-pf-Bb5Iqx6ccvVA67gLVT-G4Zl3Zl5FPUZS4d7V6rk4
certificatesigningrequest "node-csr-pf-Bb5Iqx6ccvVA67gLVT-G4Zl3Zl5FPUZS4d7V6rk4" approved

## 配置 kube-proxy

## 创建 kube-proxy 证书

cd /opt/ssl
vi kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

## 生成 kube-proxy 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
  -ca-key=/etc/kubernetes/ssl/ca-key.pem \
  -config=/opt/ssl/config.json \
  -profile=kubernetes  kube-proxy-csr.json | /opt/local/cfssl/cfssljson -bare kube-proxy

# 查看生成

ls kube-proxy*
kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem

# 拷贝到目录

cp kube-proxy*.pem /etc/kubernetes/ssl/
rsync -a /etc/kubernetes/ 10.0.1.158:/etc/kubernetes/
rsync -a /etc/kubernetes/ 10.0.1.159:/etc/kubernetes/

## 创建 kube-proxy kubeconfig 文件

> server 配置为各自本机IP

# 配置集群

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=https://127.0.0.1:6443 \
  --kubeconfig=kube-proxy.kubeconfig

# 配置客户端认证

kubectl config set-credentials kube-proxy \
  --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
  --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

# 配置关联

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

# 配置默认关联

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

# 拷贝到目录

mv kube-proxy.kubeconfig /etc/kubernetes/

## 创建 kube-proxy.service 文件

> 配置为各自的 IP

[[email protected] ~]# cat /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --bind-address=10.0.1.159 \
  --hostname-override=k8s-m-159 \
  --cluster-cidr=10.254.0.0/16 \
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \
  --logtostderr=true \
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

## 启动 kube-proxy

mkdir /var/lib/kube-proxy
systemctl daemon-reload
systemctl enable kube-proxy
systemctl start kube-proxy
systemctl status kube-prox

# 如果报错请使用

journalctl -f -t kube-proxy  和 journalctl -u kube-proxy 来定位问题

相关文章