首先感謝這份部落格 https://my.oschina.net/u/4197945/blog/15510668 作者：運維有術星主
參考KubeSphere官網文件：https://www.kubesphere.io/zh/docs/v3.4/devops-user-guide/how-to-use/pipelines/create-a-pipeline-using-graphical-editing-panel/
此份文件記錄配置過程的注意事項
1.叢集新增節點，配置config檔案，最好是用初始化時候的配置檔案。
注意點：官方文件，新增新節點 https://www.kubesphere.io/zh/docs/v3.4/installing-on-linux/cluster-operation/add-new-nodes/
用此方法生產的配置文件，少了一些配置引數，比如是kubernetes執行時是docker, 還是containerd，預設會安裝docker的執行時
所以我推薦使用初始化安裝時候的配置檔案
apiVersion: kubekey.kubesphere.io/v1alpha2 kind: Cluster metadata: name: sample spec: hosts: - {name: ksmaster01, address: 192.168.30.204, internalAddress: 192.168.30.204, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc7"} - {name: ksmaster02, address: 192.168.30.205, internalAddress: 192.168.30.205, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc8"} - {name: ksmaster03, address: 192.168.30.206, internalAddress: 192.168.30.206, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc9"} - {name: ksworker04, address: 192.168.30.212, internalAddress: 192.168.30.212, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc13"} roleGroups: etcd: - ksmaster01 - ksmaster02 - ksmaster03 control-plane: - ksmaster01 - ksmaster02 - ksmaster03 worker: - ksworker04 controlPlaneEndpoint: ## Internal loadbalancer for apiservers # internalLoadbalancer: haproxy domain: lb.kubesphere.local address: "192.168.30.203" port: 6443 kubernetes: version: v1.23.17 clusterName: cluster.local autoRenewCerts: true containerManager: containerd etcd: type: kubekey network: plugin: calico kubePodsCIDR: 10.233.64.0/18 kubeServiceCIDR: 10.233.0.0/18 ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni multusCNI: enabled: false registry: auths: "harbor.emergen.cn": username: admin password: nt5Hw7T+FpAkQ4za/vBb skipTLSVerify: false plainHTTP: false privateRegistry: "harbor.emergen.cn/kubesphereio" namespaceOverride: "" registryMirrors: ["hub.deeprobe.online"] insecureRegistries: [] addons: [] --- apiVersion: installer.kubesphere.io/v1alpha1 kind: ClusterConfiguration metadata: name: ks-installer namespace: kubesphere-system labels: version: v3.4.1 spec: persistence: storageClass: "" authentication: jwtSecret: "" local_registry: "" # dev_tag: "" etcd: monitoring: false endpointIps: localhost port: 2379 tlsEnable: true common: core: console: enableMultiLogin: true port: 30880 type: NodePort # apiserver: # resources: {} # controllerManager: # resources: {} redis: enabled: false enableHA: false volumeSize: 2Gi openldap: enabled: false volumeSize: 2Gi minio: volumeSize: 20Gi monitoring: # type: external endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 GPUMonitoring: enabled: false gpu: kinds: - resourceName: "nvidia.com/gpu" resourceType: "GPU" default: true es: # master: # volumeSize: 4Gi # replicas: 1 # resources: {} # data: # volumeSize: 20Gi # replicas: 1 # resources: {} enabled: false logMaxAge: 7 elkPrefix: logstash basicAuth: enabled: false username: "" password: "" externalElasticsearchHost: "" externalElasticsearchPort: "" opensearch: # master: # volumeSize: 4Gi # replicas: 1 # resources: {} # data: # volumeSize: 20Gi # replicas: 1 # resources: {} enabled: true logMaxAge: 7 opensearchPrefix: whizard basicAuth: enabled: true username: "admin" password: "admin" externalOpensearchHost: "" externalOpensearchPort: "" dashboard: enabled: true alerting: enabled: true # thanosruler: # replicas: 1 # resources: {} auditing: enabled: true # operator: # resources: {} # webhook: # resources: {} devops: enabled: false jenkinsCpuReq: 0.5 jenkinsCpuLim: 1 jenkinsMemoryReq: 4Gi jenkinsMemoryLim: 4Gi jenkinsVolumeSize: 16Gi events: enabled: true # operator: # resources: {} # exporter: # resources: {} ruler: enabled: true replicas: 2 # resources: {} logging: enabled: true logsidecar: enabled: true replicas: 2 # resources: {} metrics_server: enabled: true monitoring: storageClass: "" node_exporter: port: 9100 # resources: {} # kube_rbac_proxy: # resources: {} # kube_state_metrics: # resources: {} prometheus: # replicas: 1 volumeSize: 40Gi # resources: {} # operator: # resources: {} # alertmanager: # replicas: 1 # resources: {} # notification_manager: # resources: {} # operator: # resources: {} # proxy: # resources: {} gpu: nvidia_dcgm_exporter: enabled: false # resources: {} multicluster: clusterRole: none network: networkpolicy: enabled: false ippool: type: calico topology: type: weave-scope openpitrix: store: enabled: true servicemesh: enabled: false istio: components: ingressGateways: - name: istio-ingressgateway enabled: false cni: enabled: false edgeruntime: enabled: true kubeedge: enabled: true cloudCore: cloudHub: advertiseAddress: - 192.168.30.203 service: cloudhubNodePort: "30000" cloudhubQuicNodePort: "30001" cloudhubHttpsNodePort: "30002" cloudstreamNodePort: "30003" tunnelNodePort: "30004" # resources: {} # hostNetWork: false iptables-manager: enabled: true mode: "external" # resources: {} # edgeService: # resources: {} gatekeeper: enabled: false # controller_manager: # resources: {} # audit: # resources: {} terminal: timeout: 600
export KKZONE=cn
./kk add nodes -f config-nodes-add.yaml # 執行前，先執行第二步驟，確保node上已經有必須要的元件
2.配置新節點
2.1 yum 源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo-history wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo yum install -y wget yum clean all yum makecache
2.2 安裝依賴元件，看官網文件中，這幾個元件是必須的
yum -y install socat ipset conntrack ebtables ipvsadm
3.配置流水線-netcore環境
專案是由netcore語言開發
3.1 配置字典 jenkins-casc-config。找到 jenkins_user.yaml
新增 dotnetcore 容器環境,新增後等待一分鐘左右，流水線才會出現環境配置
- name: "dotnetcore" namespace: "kubesphere-devops-worker" label: "dotnetcore" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "dotnetcore" image: "harbor.emergen.cn/library/dotnet-emergen:3-5-6" # 指定自定義dotnet映象 command: "cat" args: "" ttyEnabled: true privileged: true resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" alwaysPullImage: true - name: "jnlp" #image: "jenkins/jnlp-slave:3.27-1" image: "harbor.emergen.cn/kubesphereio/jenkins/inbound-agent:4.10-2" #command: "jenkins-slave" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" imagePullSecrets: # 指定私有倉庫憑證 - name: harborsecret workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: # 將nuget包快取持久化到hostPath hostPath: "/var/data/jenkins_nuget_cache" mountPath: "/root/.nuget" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "dotnetcore" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" volumeMounts: - name: config-volume mountPath: /root/.nuget/NuGet/NuGet.Config subPath: NuGet.Config volumes: - name: config-volume configMap: name: ks-devops-agent items: - key: NugetSetting path: NuGet.Config securityContext: fsGroup: 1000 - name: "dotnetcore2-emergen" # 自定義 Jenkins Agent 的名稱。 label: "dotnetcore2-emergen" # 自定義 Jenkins Agent 的標籤。若要指定多個標籤，請用空格來分隔標籤。 inheritFrom: "dotnetcore" # 該自定義 Jenkins Agent 所繼承的現有容器組模板的名稱。 containers: - name: "dotnetcore" # 該自定義 Jenkins Agent 所繼承的現有容器組模板中指定的容器名稱。 image: "harbor.emergen.cn/library/dotnet-emergen:2.2" # 可以使用自己的映象。 imagePullSecrets: - name: harborsecret - name: "dotnetcore6-emergen" # 自定義 Jenkins Agent 的名稱。 label: "dotnetcore6-emergen" # 自定義 Jenkins Agent 的標籤。若要指定多個標籤，請用空格來分隔標籤。 inheritFrom: "dotnetcore" # 該自定義 Jenkins Agent 所繼承的現有容器組模板的名稱。 containers: - name: "dotnetcore" # 該自定義 Jenkins Agent 所繼承的現有容器組模板中指定的容器名稱。 image: "harbor.emergen.cn/library/dotnet-emergen:v6" # 可以使用自己的映象。 imagePullSecrets: - name: harborsecret
4. 配置流水線-pipline
注意: 涉及到 credentialsId，都是在devops 憑據中配置，其他是執行引數
pipeline { agent { node { label 'dotnetcore' } } stages { stage('拉取程式碼') { steps { git(url: "${GIT_REPOSITORY_URL}", credentialsId: 'gitlab-auth', branch: "${BRANCH}", changelog: true, poll: false) } } stage('編譯推送') { steps { container("dotnetcore"){ sh 'dotnet restore "./demo/demo.csproj" ' sh 'dotnet build ./demo/demo.csproj -c Release -o dist --force' sh 'podman build -t $REGISTRY/$HARBOR_NAMESPACE/$APP_NAME:$BUILD_NUMBER .' withCredentials([usernamePassword(credentialsId : 'harbor-auth' ,passwordVariable : 'HARBOR_PASSWORD' ,usernameVariable : 'HARBOR_USERNAME' ,)]) { sh '''echo "$HARBOR_PASSWORD" | podman login $REGISTRY -u "$HARBOR_USERNAME" --password-stdin ''' sh '''podman push $REGISTRY/$HARBOR_NAMESPACE/$APP_NAME:$BUILD_NUMBER ''' } } } } stage('部署') { steps { container ('dotnetcore') { withCredentials([ kubeconfigFile(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) { sh 'envsubst < webapi.yaml | kubectl apply -f -' } } } } } }
5. netcore環境的dockerfile
FROM registry.cn-beijing.aliyuncs.com/kubesphereio/builder-base:v3.2.2-podman RUN sed -e 's|^mirrorlist=|#mirrorlist=|g' \ -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos|g' \ -i.bak \ /etc/yum.repos.d/CentOS-*.repo && \ rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm && \ yum install -y dotnet-sdk-3.1 && \ yum install -y dotnet-sdk-5.0 && \ yum install -y dotnet-sdk-6.0 && \ yum clean all #RUN dotnet tool install --global dotnet-sonarscanner --version 5.0.4 #ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/bin/sonar-scanner-3.3.0.1492-linux/bin:/root/.nuget/tools:/root/.dotnet/tools ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.nuget/tools:/root/.dotnet/tools CMD ["dotnet", "--list-sdks"]