k8s二進位制部署

1.準備環境
a.關閉防火牆
systemctl stop firewalld
systemctl disable firewalld

#檢查
systemctl status firewalld
b.關閉selinx
vi /etc/selinux/config

#檢查
getenforce

c.關閉swap
vi /etc/fstab

#檢查
free

d.配置主機與IP對應關係
vi /etc/hosts

e.將橋接IPV4流程傳遞到iptables
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl --system

#檢查
iptables -vnL

2.kubeadmin模式安裝k8s
a.所有節點安裝docker/kubeadm/kubelet

#新增阿里雲docker源
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#列出所有docker可安裝版本
yum list docker-ce --showduplicates | sort -r
#安裝docker
yum install docker-ce-18.06.1.ce-3.el7
systemctl enable docker
systemctl start docker

#新增阿里雲YUM軟體源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安裝kubelet、kubeadm和kubectl
yum remove -y kubelet-1.13.1 kubeadm-1.13.1 kubectl-1.13.1
yum install -y kubelet-1.13.3 kubeadm-1.13.3 kubectl-1.13.3
systemctl enable kubelet

#部署master節點
kubeadm init --kubernetes-version=1.13.1 \
--apiserver-advertise-address=192.168.43.205 \
--image-repository registry.aliyuncs.com/google_containers \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.254.0.0/16

#部署網路外掛
wget https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml

#部署2個node節點
kubeadm join 192.168.43.100:6443 --token 3n01sl.qjsscml383uzldzn --discovery-token-ca-cert-hash sha256:aa728efa0d4cacf2eb064f05ea3535152a0432a5a18c69cdf312bd646830f1f1

#檢查環境
kubectl get node
kubectl get cs

kubectl create deployment nginx --image=nginx
kubeclt expose deployment nginx --port=80 --type=NodePort

#部署dashboard
kubectl apply -f kubernetes-dashboard.yaml

#建立dashboard登入使用者
kubectl create serviceaccount dashboard-admin -n kube-system
kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin

kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admim/{print $1}')

2.k8s二進位制模式安裝
a.環境規劃
master節點：192.168.43.205 安裝（kube-apiserver，kube-controller-manager，kube-scheduler，etcd）
node節點：192.168.43.206 安裝（kubelet，kube-proxy，docker，etcd）
node節點：192.168.43.207 安裝（kubelet，kube-proxy，docker，etcd）

kubernetes版本：1.9
docker版本：17.12-ce
etcd版本：3.0

作業系統：centos7.4_x64  cpu 2核+2G記憶體+30G磁碟

b.安裝docker
yum remove docker \
    docker-client \
    docker-client-latest \
    docker-common \
    docker-latest \
    docker-latest-logrotate \
    docker-logrotate \
    docker-engine
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum list docker-ce --showduplicates | sort -r

yum install -y docker-ce-17.12.0.ce-1.el7.centos

cat << EOF > /etc/docker/daemon.json
{
  "registry-mirrors":["http://registry.docker-cn.com"],
  "insecure-registries":["192.168.0.210:5000"]
}
EOF

#docker新增阿里庫
https://cr.console.aliyun.com/cn-hangzhou/instances/mirrors

cat << EOF > /etc/docker/daemon.json
{
  "registry-mirrors":["https://live9m0e.mirror.aliyuncs.com"]
}
EOF

systemctl daemon-reload
systemctl restart docker

#啟動docker容器
systemctl start docker
systemctl enable docker

c.自籤TLS證書
#證書分佈情況
etcd元件：ca.pem,server.pem,server-key.pem
kube-apiserver元件：ca.pem,server.pem,server-key.pem
kubelet元件：ca.pem,ca-key.pem
kube-proxy元件：ca.pem,kube-proxy.pem,kube-proxy-key.pem
kubectl元件：ca.pem,admin.pem,admin-key.pem

#證書安裝
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

mkdir /root/ssl
cd /root/ssl

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json << EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

cat > server-csr.json << EOF
{
    "CN": "kubernetes",
    "hosts": [
      "127.0.0.1",
      "192.168.43.205",
      "192.168.43.206",
      "192.168.43.207",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [{
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

cat > admin-csr.json << EOF
{
    "CN": "admin",
    "hosts": [],
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [{
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

cat > kube-proxy-csr.json << EOF
{
    "CN": "system:kube-proxy",
    "hosts": [],
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [{
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

d.部署ETCD叢集
#建立初始目錄
mkdir /opt/kubernetes/
mkdir /opt/kubernetes/{bin,cfg,ssl}

#下載etcd二進位制包並解壓
tar xzvf etcd-v3.2.12-linux-amd64.tar.gz
mv etcd /opt/kubernetes/bin/
mv etcdctl /opt/kubernetes/bin/

#建立etcd.service
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf
ExecStart=/opt/kubernetes/bin/etcd \
--name=etcd01 \
--data-dir=/var/lib/etcd/default.etcd \
--listen-peer-urls=https://192.168.43.205:2380 \
--listen-client-urls=https://192.168.43.205:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://192.168.43.205:2379 \
--initial-advertise-peer-urls=https://192.168.43.205:2380 \
--initial-cluster-token=etcd-cluster \
--initial-cluster=etcd01=https://192.168.43.205:2380,etcd02=https://192.168.43.206:2380,etcd03=https://192.168.43.207:2380 \
--initial-cluster-state=new \
--cert-file=/opt/kubernetes/ssl/server.pem \
--key-file=/opt/kubernetes/ssl/server-key.pem \
--peer-cert-file=/opt/kubernetes/ssl/server.pem \
--peer-key-file=/opt/kubernetes/ssl/server-key.pem \
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

EOF

#啟動etcd服務
systemctl start etcd.service
systemctl enable etcd.service

#檢查etcd叢集情況
/opt/kubernetes/bin/etcdctl  \
--endpoints=https://192.168.43.205:2379 \
--cert-file=/opt/kubernetes/ssl/server.pem \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--key-file=/opt/kubernetes/ssl/server-key.pem \
cluster-health

/opt/kubernetes/bin/etcdctl \
--endpoints=https://192.168.43.205:2379 \
--cert-file=/opt/kubernetes/ssl/server.pem \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--key-file=/opt/kubernetes/ssl/server-key.pem \
member list

e.部署flannel網路
#下載二進位制安裝包（節點安裝）
tar -xzvf flannel-v0.7.1-linux-amd64.tar.gz -C flannel
cp flannel/{flanneld,mk-docker-opts.sh} /opt/kubernetes/bin

#寫入分配的子網段到etcd
/opt/kubernetes/bin/etcdctl \
--endpoints="https://192.168.43.205:2379,https://192.168.43.206:2379,https://192.168.43.207:2379" \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/server.pem \
--key-file=/opt/kubernetes/ssl/server-key.pem \
set /coreos.com/network/config '{"Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
  
#flannel啟動檔案
cat > flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=/root/local/bin/flanneld \\
  -etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
  -etcd-certfile=/etc/flanneld/ssl/flanneld.pem \\
  -etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \\
  -etcd-endpoints=${ETCD_ENDPOINTS} \\
  -etcd-prefix=${FLANNEL_ETCD_PREFIX}
ExecStartPost=/root/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

systemctl start flannel.service 
systemctl enable flannel.service

#修改docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target

systemctl daemon-reload 
systemctl restart docker.service 

f.建立node節點kubeconfig檔案

#建立TLS自動獲取證書憑證
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

#建立節點bootstrap.kubeconfig
/opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.43.205:6443 \
--kubeconfig=bootstrap.kubeconfig

#設定客戶端認證引數
/opt/kubernetes/bin/kubectl config set-credentials kubelet-bootstrap \
--token=c64c942862608063f2eabff13077ef7e \
--kubeconfig=bootstrap.kubeconfig

#設定上下文引數
/opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig

#設定預設上下文
/opt/kubernetes/bin/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

#建立節點kube-proxy kubeconfig檔案
/opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.43.205:6443 \
--kubeconfig=kube-proxy.kubeconfig

/opt/kubernetes/bin/kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
--client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

/opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

/opt/kubernetes/bin/kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

g.獲取K8S二進位制包
從git上下載二進位制安裝包，並解壓

h.執行master元件
#安裝kube-apiserver
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver
ExecStart=/opt/kubernetes/bin/kube-apiserver \
--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.43.205:2379,https://192.168.43.206:2379,https://192.168.43.207:2379 \
--insecure-bind-address=127.0.0.1 \
--bind-address=192.168.43.205 \
--insecure-port=8080 \
--secure-port=6443 \
--advertise-address=192.168.43.205 \
--allow-privileged=true \
--service-cluster-ip-range=10.10.10.0/24 \
--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/kubernetes/ssl/ca.pem \
--etcd-certfile=/opt/kubernetes/ssl/server.pem \
--etcd-keyfile=/opt/kubernetes/ssl/server-key.pem
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl start kube-apiserver
systemctl enable kube-apiserver

#安裝kube-controller-manager
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/opt/kubernetes/bin/kube-controller-manager \
--logtostderr=true \
--v=4 \
--master=127.0.0.1:8080 \
--leader-elect=true \
--address=127.0.0.1 \
--service-cluster-ip-range=10.10.10.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \
--root-ca-file=/opt/kubernetes/ssl/ca.pem
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl start kube-controller-manager.service
systemctl enable kube-controller-manager.service

#安裝kube-scheduler 
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/opt/kubernetes/bin/kube-scheduler \
--logtostderr=true \
--v=4 \
--master=127.0.0.1:8080 \
--leader-elect
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl start kube-scheduler.service
systemctl enable kube-scheduler.service

#檢查master是否安裝成功
kubectl get cs

i.執行node元件
#安裝kubelet
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \
--logtostderr=true \
--v=4 \
--address=192.168.43.206 \
--hostname-override=192.168.43.206 \
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
--experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
--cert-dir=/opt/kubernetes/ssl \
--allow-privileged=true \
--cluster-dns=10.10.10.2 \
--cluster-domain=cluster.local \
--fail-swap-on=false \
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

#給使用者kubelet-bootstrap授權獲取證書
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

#獲取證書請求情況
kubectl get csr
#同意獲取證書
kubectl certificate approve node-csr-1WbGGKYvoiLOLYQXZDeXWaFUAOy-wCF5kXrN0l9Pd5c
kubectl certificate approve node-csr-uC1bOZEvKhWJ4Zgsy2UjS_6ATRR4oPgy6xfX73Ulj_Q

systemctl start kubelet.service
systemctl enable kubelet.service

#安裝kube-proxy
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kub-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \
--logtostderr=true \
--v=4 \
--hostname-override=192.168.43.206 \
--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig 
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

systemctl start kube-proxy.service
systemctl enable kube-proxy.service

j.查詢叢集狀態
kubectl get cs
kubectl get nodes

k.啟動測試示例
kubectl run nginx --image=nginx --replicas=3
kubectl get pod
kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
kubectl get svc nginx

https://archive.kernel.org/centos-vault/7.4.1708/isos/x86_64/

rpm -qa | grep iptables
rpm -ivh iptables-1.4.21-24.el7.x86_64.rpm

yum remove iptables
yum install -y iptables-1.4.21-24.el7.x86_64

l.部署Dashboard