一、etcd备份:

  • 脚本中对k8s做了备份,但是没有对etcd数据做备份,需要对etcd数据做备份。
  • 参考etcd(V3版api)备份和恢复
    二 、master节点k8s证书更换(分别在每个master上执行即可)
  • 脚本最好放在一个空目录下执行
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    #!/bin/bash
    if [ ! -d "/root/tmp/" ]; then
    mkdir /root/tmp/
    fi
    cp -rf /etc/kubernetes /root/tmp/kubernetes_`date '+%Y%m%d_%H.%M.%S'`
    #更换apiserver证书
    cat <<EOF>ssl.conf
    [req]
    req_extensions = v3_req
    distinguished_name = req_distinguished_name
    [req_distinguished_name]
    [v3_req]
    keyUsage =critical, digitalSignature, keyEncipherment
    extendedKeyUsage = TLS Web Server Authentication, TLS Web Client Authentication
    subjectAltName = @alt_names
    [alt_names]
    EOF
    ip=1
    dns=1
    for aa in `openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -text |grep "DNS"|sed "s/,/\n/g"|grep "DNS"|tr -d " "`; do echo $aa| sed "s/DNS:/DNS.$dns = /" >> ssl.conf;let dns+=1; done
    for aa in `openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -text |grep "IP Address"|sed "s/,/\n/g"|grep "IP Address"|tr -d " " `;do echo $aa| sed "s/IPAddress:/IP.$ip = /" >> ssl.conf;let ip+=1; done
    openssl req -new -key /etc/kubernetes/pki/apiserver.key -out apiserver.csr -subj "/CN=kube-apiserver" -config ssl.conf
    openssl x509 -req -in apiserver.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out apiserver.crt -days 10950 -extensions v3_req -extfile ssl.conf
    mv /etc/kubernetes/pki/apiserver.crt /tmp
    cp apiserver.crt /etc/kubernetes/pki/

    #更换apiserver-kubelet-client证书
    cat <<EOF>client.conf
    [ v3_ca ]
    keyUsage = critical, digitalSignature, keyEncipherment
    extendedKeyUsage = clientAuth
    EOF


    openssl genrsa -out admin.key 2048
    openssl req -new -key admin.key -out admin.csr -subj "/O=system:masters/CN=kube-apiserver-kubelet-client"
    openssl x509 -req -in admin.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -sha256 -out admin.crt -extensions v3_ca -extfile client.conf -days 3650
    mv /etc/kubernetes/pki/apiserver-kubelet-client.key /tmp
    mv /etc/kubernetes/pki/apiserver-kubelet-client.crt /tmp
    mv admin.key apiserver-kubelet-client.key
    mv admin.crt apiserver-kubelet-client.crt
    cp apiserver-kubelet-client.key /etc/kubernetes/pki/
    cp apiserver-kubelet-client.crt /etc/kubernetes/pki/
    kubectl delete pod -n kube-system kube-apiserver-$HOSTNAME
    docker ps |grep "kube-apiserver" | awk '{print $1}' | xargs docker rm -f


    #更换front-proxy-client证书
    openssl genrsa -out client.key 2048
    openssl req -new -key client.key -out client.csr -subj "/O=system:masters/CN=front-proxy-client"
    openssl x509 -req -in client.csr -CA /etc/kubernetes/pki/front-proxy-ca.crt -CAkey /etc/kubernetes/pki/front-proxy-ca.key -CAcreateserial -sha256 -out client.crt -extensions v3_ca -extfile client.conf -days 3650
    mv /etc/kubernetes/pki/front-proxy-client.key /tmp
    mv /etc/kubernetes/pki/front-proxy-client.crt /tmp
    mv client.key front-proxy-client.key
    mv client.crt front-proxy-client.crt
    cp front-proxy-client.key /etc/kubernetes/pki/
    cp front-proxy-client.crt /etc/kubernetes/pki/
    sleep 8


    #更换Kubelet证书
    #Master节点直接执行此脚本
    cat <<EOF>kube.conf
    [ v3_ca ]
    keyUsage = critical, digitalSignature, keyEncipherment
    extendedKeyUsage = clientAuth
    EOF


    openssl genrsa -out kubelet.key 2048
    openssl req -new -key kubelet.key -out kubelet.csr -subj "/O=system:nodes/CN=system:node:$HOSTNAME"
    openssl x509 -req -in kubelet.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -sha256 -out kubelet.crt -extensions v3_ca -extfile kube.conf -days 3650
    base64 kubelet.crt | tr -d '\n' > kubelet-crt-data
    base64 kubelet.key | tr -d '\n' > kubelet-key-data
    KUBELET_CRT_DATA=$(cat kubelet-crt-data)
    KUBELET_KEY_DATA=$(cat kubelet-key-data)
    sed -i \
    -e 's/client-certifica..*:.*/'"client-certificate-data: $KUBELET_CRT_DATA"'/' \
    -e 's/client-k..*:.*/'"client-key-data: $KUBELET_KEY_DATA"'/' \
    /etc/kubernetes/kubelet.conf

    #controller证书更换
    openssl genrsa -out controller.key 2048
    openssl req -new -key controller.key -out controller.csr -subj "/CN=system:kube-controller-manager"
    openssl x509 -req -in controller.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -sha256 -out controller.crt -extensions v3_ca -extfile client.conf -days 3650
    base64 controller.crt | tr -d '\n' > controller-crt-data
    base64 controller.key | tr -d '\n' > controller-key-data

    CONTROLLER_CRT_DATA=$(cat controller-crt-data)
    CONTROLLER_KEY_DATA=$(cat controller-key-data)
    sed -i \
    -e 's/client-certifica..*:.*/'"client-certificate-data: $CONTROLLER_CRT_DATA"'/' \
    -e 's/client-k..*:.*/'"client-key-data: $CONTROLLER_KEY_DATA"'/' \
    /etc/kubernetes/controller-manager.conf
    kubectl delete pod -n kube-system kube-controller-manager-$HOSTNAME
    docker ps |grep "k8s_kube-controller-manager" |awk '{print $1}' | xargs docker rm -f

    #scheduler证书更换
    openssl genrsa -out scheduler.key 2048
    openssl req -new -key scheduler.key -out scheduler.csr -subj "/CN=system:kube-scheduler"
    openssl x509 -req -in scheduler.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -sha256 -out scheduler.crt -extensions v3_ca -extfile client.conf -days 3650
    base64 scheduler.crt | tr -d '\n' > scheduler-crt-data
    base64 scheduler.key | tr -d '\n' > scheduler-key-data

    SCHEDULER_CRT_DATA=$(cat scheduler-crt-data)
    SCHEDULER_KEY_DATA=$(cat scheduler-key-data)
    sed -i \
    -e 's/client-certifica..*:.*/'"client-certificate-data: $SCHEDULER_CRT_DATA"'/' \
    -e 's/client-k..*:.*/'"client-key-data: $SCHEDULER_KEY_DATA"'/' \
    /etc/kubernetes/scheduler.conf
    kubectl delete pod -n kube-system kube-scheduler-$HOSTNAME
    docker ps |grep "k8s_kube-scheduler" |awk '{print $1}' | xargs docker rm -f


    systemctl restart kubelet

    echo -e "\033[5;33;40m master单节点证书签发成功\033[0m \n"

    echo -e "\033[5;33;40m 验证证书,请仔细查看证书签发时间\033[0m \n"
    echo "--------------------------------"
    echo "apiserver证书"
    echo `openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -dates`
    echo "--------------------------------"
    echo "kubelet证书"
    cat /etc/kubernetes/kubelet.conf |grep client-certificate-data |awk -F: '{print $2}' | awk '{print $1}' | base64 -d > a.crt
    echo `openssl x509 -in a.crt -noout -dates`
    echo "--------------------------------"
    echo "controller-manager证书"
    cat /etc/kubernetes/controller-manager.conf |grep client-certificate-data |awk -F: '{print $2}' | awk '{print $1}' | base64 -d > a.crt
    echo `openssl x509 -in a.crt -noout -dates`
    echo "--------------------------------"
    echo "scheduler证书"
    cat /etc/kubernetes/scheduler.conf |grep client-certificate-data |awk -F: '{print $2}' | awk '{print $1}' | base64 -d > a.crt
    echo `openssl x509 -in a.crt -noout -dates`
    echo "--------------------------------"
    echo "apiserver-kubelet-client证书"
    echo `openssl x509 -in /etc/kubernetes/pki/apiserver-kubelet-client.crt -noout -dates`
    echo "--------------------------------"
    echo "front-proxy-client证书"
    echo `openssl x509 -in /etc/kubernetes/pki/front-proxy-client.crt -noout -dates`
    rm -rf a.crt
    三、master节点etcd证书更换(分别在每个master上执行即可)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
if [ ! -d "/root/tmp/" ]; then
mkdir /root/tmp/
fi
cp -rf /etc/kubernetes/pki/etcd /root/tmp/etcd_`date '+%Y%m%d_%H.%M.%S'`


#etcd的healthcheck-client-client证书更换
cat <<EOF>healthcheck.conf
[ v3_ca ]
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth, serverAuth,TLS Web Server Authentication, TLS Web Client Authentication
EOF


#openssl genrsa -out healthcheck-client.key 2048
openssl req -new -key /etc/kubernetes/pki/etcd/healthcheck-client.key -out healthcheck-client.csr -subj "/O=system:masters/CN=kube-etcd-healthcheck-client"
openssl x509 -req -in healthcheck-client.csr -CA /etc/kubernetes/pki/etcd/ca.crt -CAkey /etc/kubernetes/pki/etcd/ca.key -CAcreateserial -sha256 -out healthcheck-client.crt -extensions v3_ca -extfile healthcheck.conf -days 3650
mv /etc/kubernetes/pki/etcd/healthcheck-client.crt /tmp
cp healthcheck-client.crt /etc/kubernetes/pki/etcd/


#etcd的peer证书更换

cat <<EOF>peer-ssl.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[v3_req]
keyUsage =critical, digitalSignature, keyEncipherment
extendedKeyUsage = TLS Web Server Authentication, TLS Web Client Authentication
subjectAltName = @alt_names
[alt_names]
EOF
ippeer=1
dnspeer=1
for bb in `openssl x509 -in /etc/kubernetes/pki/etcd/peer.crt -noout -text |grep "DNS"|sed "s/,/\n/g"|grep "DNS"|tr -d " "`; do echo $bb| sed "s/DNS:/DNS.$dnspeer = /" >> peer-ssl.conf;let dnspeer+=1; done
for bb in `openssl x509 -in /etc/kubernetes/pki/etcd/peer.crt -noout -text |grep "IP Address"|sed "s/,/\n/g"|grep "IP Address"|tr -d " " `;do echo $bb| sed "s/IPAddress:/IP.$ippeer = /" >> peer-ssl.conf;let ippeer+=1; done
openssl req -new -key /etc/kubernetes/pki/etcd/peer.key -out peer.csr -subj "/CN=$HOSTNAME" -config peer-ssl.conf
openssl x509 -req -in peer.csr -CA /etc/kubernetes/pki/etcd/ca.crt -CAkey /etc/kubernetes/pki/etcd/ca.key -CAcreateserial -out peer.crt -days 3650 -extensions v3_req -extfile peer-ssl.conf
mv /etc/kubernetes/pki/etcd/peer.crt /tmp
cp peer.crt /etc/kubernetes/pki/etcd


#etcd的server证书更换
cat <<EOF>server-ssl.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[v3_req]
keyUsage =critical, digitalSignature, keyEncipherment
extendedKeyUsage = TLS Web Server Authentication, TLS Web Client Authentication
subjectAltName = @alt_names
[alt_names]
EOF
ipserver=1
dnsserver=1
for cc in `openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -text |grep "DNS"|sed "s/,/\n/g"|grep "DNS"|tr -d " "`; do echo $cc| sed "s/DNS:/DNS.$dnsserver = /" >> server-ssl.conf;let dnsserver+=1; done
for cc in `openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -text |grep "IP Address"|sed "s/,/\n/g"|grep "IP Address"|tr -d " " `;do echo $cc| sed "s/IPAddress:/IP.$ipserver = /" >> server-ssl.conf;let ipserver+=1; done
openssl req -new -key /etc/kubernetes/pki/etcd/server.key -out server.csr -subj "/CN=$HOSTNAME" -config server-ssl.conf
openssl x509 -req -in server.csr -CA /etc/kubernetes/pki/etcd/ca.crt -CAkey /etc/kubernetes/pki/etcd/ca.key -CAcreateserial -out server.crt -days 3650 -extensions v3_req -extfile server-ssl.conf
mv /etc/kubernetes/pki/etcd/server.crt /tmp
cp server.crt /etc/kubernetes/pki/etcd
kubectl delete pod -n kube-system etcd-$HOSTNAME
docker ps |grep etcd | awk '{print $1}' | xargs docker rm -f
sleep 2


echo -e "\033[5;33;40m master单节点etcd证书签发成功\033[0m \n"

echo -e "\033[5;33;40m 验证证书,请仔细查看证书签发时间\033[0m \n"


echo "--------------------------------"
echo "etcd-healthcheck-client证书"
echo `openssl x509 -in /etc/kubernetes/pki/etcd/healthcheck-client.crt -noout -dates`
echo "--------------------------------"
echo "etcd-peer证书"
echo `openssl x509 -in /etc/kubernetes/pki/etcd/peer.crt -noout -dates`
echo "--------------------------------"
echo "etcd-server证书"
echo `openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -dates`

四、slave的kubelet证书升级(分别在每个slave上执行即可)

  • 把master节点的/etc/kubernetes/pki目录下的ca.key拷贝到slave的/etc/kubernetes/pki目录下
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
/bin/bash
if [ ! -d "/root/tmp/" ]; then
mkdir /root/tmp/
fi
cp -rf /etc/kubernetes /root/tmp/kubernetes_`date '+%Y%m%d_%H.%M.%S'`


cat <<EOF>kube.conf
[ v3_ca ]
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
EOF


openssl genrsa -out kubelet.key 2048
openssl req -new -key kubelet.key -out kubelet.csr -subj "/O=system:nodes/CN=system:node:$HOSTNAME"
openssl x509 -req -in kubelet.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -sha256 -out kubelet.crt -extensions v3_ca -extfile kube.conf -days 3650
base64 kubelet.crt | tr -d '\n' > kubelet-crt-data
base64 kubelet.key | tr -d '\n' > kubelet-key-data
KUBELET_CRT_DATA=$(cat kubelet-crt-data)
KUBELET_KEY_DATA=$(cat kubelet-key-data)
sed -i \
-e 's/client-certifica..*:.*/'"client-certificate-data: $KUBELET_CRT_DATA"'/' \
-e 's/client-k..*:.*/'"client-key-data: $KUBELET_KEY_DATA"'/' \
/etc/kubernetes/kubelet.conf
systemctl restart kubelet




echo -e "\033[5;33;40m slave单节点kubelet证书签发成功\033[0m \n"

echo -e "\033[5;33;40m 验证证书,请仔细查看证书签发时间\033[0m \n"

cat /etc/kubernetes/kubelet.conf |grep client-certificate-data |awk -F: '{print $2}' | awk '{print $1}' | base64 -d > a.crt
echo `openssl x509 -in a.crt -noout -dates`
rm -rf a.crt

五、删除dns、proxy、flannel的pod

1
2
3
kubectl get pod -n kube-system |grep proxy |awk '{print $1}'|xargs kubectl delete pod -n kube-system
kubectl get pod -n kube-system |grep dns |awk '{print $1}'|xargs kubectl delete pod -n kube-system
kubectl get pod -n kube-system |grep flannel |awk '{print $1}'|xargs kubectl delete pod -n kube-system

六、功能验证