CICD: jenkins发布服务到kubernetes

jenkins-cicd1-and-helm

jenkins 页面构建通过 helm 发布到 k8s 集群中

jenkins job 选项参数

branch: master
SVC_ENV: prod
PORT: 8080
JAVA_OPT: 空
Nacos_Server: http://nacos-headless.nacos.svc:8848
Nacos_Namespace: 00f675efda
replicas: 2
k8s_namespace: xxx-dev
skywalking: xxx:11800

流水线

  • SCM
    • git
      • repository url 代码仓库
      • 脚本路径: deploy/Jenkinsfile

deploy/Jenkinsfile

#!groovy
node('master') {
    def jobname = env.JOB_BASE_NAME
    def name = jobname.split(/(dev-|qa-|prod-)/)[1]
    def workdir = env.WORKSPACE
    def port = env.PORT
    def nacosserver = env.Nacos_Server
    def nacosns = env.Nacos_Namespace
    def dockerid = env.BUILD_TAG
    def appenv = env.SVC_ENV
    def k8sns = env.k8s_namespace
    def k8sreplicas = env.replicas
    def skywalking = env.skywalking

    stage('代码获取') {
        dir("${workdir}") {
            checkout scm
        }
    }

    stage('构建代码') {
        sh "mvn clean package"
        sh "cp ${name}/target/${name}.jar deploy"
    }

    stage('构建Docker镜像') {
        docker.withRegistry('https://registry-vpc.cn-beijing.aliyuncs.com', 'e87917f6-1219-4d80-8c3e-d054ecbe268d') {
            def Image = docker.build("registry-vpc.cn-beijing.aliyuncs.com/xxx-${appenv}/${name}:latest", "./deploy --build-arg JAR_FILE=${name}.jar --build-arg SERVER_NAME=${name} --build-arg PORT=${port} --build-arg ENV=${appenv} --build-arg NACOS_SERVER=${nacosserver} --build-arg NACOS_NS=${nacosns} --build-arg SKYWALKING_SERVER=${skywalking}")
            Image.push("${dockerid}")
            Image.push("latest")
        }
    }

    try {
        stage('卸载服务') {
            sh "helm --kubeconfig /root/.kube/cici/${appenv}-config uninstall ${name} -n ${k8sns}"
            sh "sed -i 's/service-name/${name}/g' deploy/helm/Chart.yaml"
        }
    } catch (Exception e) {
        echo "continue"
    }

    stage('发布服务') {
        sh "helm --kubeconfig /root/.kube/cici/${appenv}-config upgrade ${name} deploy/helm \
             --install \
             --timeout 5m0s \
             --namespace ${k8sns} \
             --set service.port=${port} \
             --set name=${name} \
             --set replicas=${k8sreplicas} \
             --set deployment.image.registry=registry-vpc.cn-beijing.aliyuncs.com/cici-${appenv}/${name} \
             --set deployment.image.version=${dockerid}"
    }

    stage('服务列表') {
        sh "helm --kubeconfig /root/.kube/cici/${appenv}-config list -A | grep ${name}"
    }
}

deploy/Dockerfile 

FROM registry-vpc.cn-beijing.aliyuncs.com/cici-dev/openjdk8:oap-latest

ARG JAR_FILE 
ARG PORT
ARG ENV
ARG NACOS_SERVER
ARG NACOS_NS
ARG JAVA_OPT
ARG SKYWALKING_SERVER 
ARG SERVER_NAME

ENV TZ=Asia/Shanghai
ENV JAR_FILE=${JAR_FILE}
ENV PORT=${PORT}
ENV ENV=${ENV}
ENV NACOS_SERVER=${NACOS_SERVER}
ENV NACOS_NS=${NACOS_NS}
ENV JAVA_OPT=${JAVA_OPT}
ENV SKYWALKING_SERVER=${SKYWALKING_SERVER}
ENV SERVER_NAME=${SERVER_NAME}

WORKDIR /app

COPY ${JAR_FILE} . 
COPY down_nacos.sh .

EXPOSE ${PORT}

ENTRYPOINT exec java ${JAVA_OPT} -javaagent:/opt/apache-skywalking-apm-bin/agent/skywalking-agent.jar -Dskywalking.agent.service_name=${SERVER_NAME} -Dskywalking.collector.backend_service=${SKYWALKING_SERVER} -jar ${JAR_FILE} --server.port=${PORT} --spring.profiles.active=${ENV} --spring.cloud.nacos.config.server-addr=${NACOS_SERVER} --spring.cloud.nacos.config.namespace=${NACOS_NS}

deploy/down_nacos.sh 

ip=`ifconfig  | grep inet | awk '{print $2}' | grep -v "127.0.0.1"`
serviceName=`ls /app | grep *.jar | awk -F '.' '{print $1}'`
port=`ps -ef | grep java | grep -v grep  | awk -F '--server.port=' '{print $2}' | awk '{print $1}'`
namespacesId=`ps -ef | grep java | grep -v grep  | awk -F '--spring.cloud.nacos.config.namespace=' '{print $2}' | awk '{print $1}'`
nacosServer=`ps -ef | grep java | grep -v grep  | awk -F '--spring.cloud.nacos.config.server-addr=' '{print $2}' | awk '{print $1}'`

curl -X PUT -d "serviceName=$serviceName" -d "ip=$ip" -d "port=$port" -d "enabled=false" -d "namespacesId=$namespacesId" $nacosServer/nacos/v1/ns/instance\?serviceName\=$serviceName\&ip\=$ip\&port\=$port\&enabled\=false\&namespaceId\=$namespacesId

sleep 30

deploy/helm/Chart.yaml 

apiVersion: v1
description: service-name service 
name: service-name-helm
version: 1.0.0

deploy/helm/values.yaml 

name: cici-gateway 

replicas: 2

service:
  port: 9999
  check: /health

deployment:
  image:
    registry: registry.cn-beijing.aliyuncs.com/cici-dev/cici-gateway 
    version: latest
    pullSecretName: cici-dev-registry

deploy/helm/templates/deployment.yaml 

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    cici.apps: {{.Values.name}}
  name: {{.Values.name}}
  namespace: {{.Values.namespace}}
spec:
  replicas: {{.Values.replicas}} 
  selector:
    matchLabels:
      cici.apps: {{.Values.name}}
  template:
    metadata:
      labels:
        cici.apps: {{.Values.name}}
    spec:
{{- with .Values.deployment.image}}
      imagePullSecrets:      
      - name: {{.pullSecretName}}
      containers:
      - image: {{ .registry}}:{{.version}}
        imagePullPolicy: {{ .pullPolicy | default "Always" | quote }}
{{- end}}
        name: {{.Values.name}}
        ports:
        - containerPort: {{.Values.service.port}} 
        readinessProbe:
          httpGet:
            path: {{.Values.service.check}}
            port: {{.Values.service.port}}
            scheme: HTTP
          initialDelaySeconds: 30
          periodSeconds: 20
          timeoutSeconds: 10
          failureThreshold: 2
          successThreshold: 1
        livenessProbe:
          httpGet:
            path: {{.Values.service.check}}
            port: {{.Values.service.port}}
            scheme: HTTP
          initialDelaySeconds: 30
          periodSeconds: 20
          timeoutSeconds: 10
          failureThreshold: 1
          successThreshold: 1
        lifecycle:
          preStop:
            exec:
              command: ["/bin/bash", "/app/down_nacos.sh"]
        resources:
          requests:
            memory: "2000Mi"
      restartPolicy: Always

deploy/helm/templates/service.yaml 

apiVersion: v1
kind: Service
metadata:
  labels:
    cici.apps: {{.Values.name}}
  name: {{.Values.name}}
  namespace: {{.Values.namespace}} 
spec:
  selector:
    cici.apps: {{.Values.name}}
  ports:
  - name: tcp 
    port: {{.Values.service.port}}
    targetPort: {{.Values.service.port}}

jenkins-cicd2

jenkins 页面构建通过 kubernetes插件 发布到 k8s 集群中

脚本式流水线与声明式流水线等价

声明式(前端) 

pipeline {
    environment {
      COMMIT_ID = ""
      imgurl = "1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/${JOB_BASE_NAME}"  //生成镜像存放镜像的仓库地址
      GIT_ADDR = "git@xxx.git/we/r.git"  //代码仓库地址
      BRANCH = 'develop' //master develop
      ENV = 'dev'
      TAG = ""                               //镜像tag,会在下面生成,这里只是定义全局变量
    }
    //全局配置
    options {
      timestamps()                     //所有输出每行都会打印时间戳
      buildDiscarder(logRotator(numToKeepStr: '20'))  //保留20个历史构建版本
    }
    agent {
        kubernetes {
            cloud 'aws-overseas'
            yaml '''
apiVersion: v1
kind: Pod
spec:
  #imagePullSecrets:
  #- name: abcd-hub
  containers:
  - name: jnlp
    image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/jnlp-slave:latest
    imagePullPolicy: IfNotPresent
    args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\']
    workingDir: /home/jenkins/agent
    tty: true
    volumeMounts:
    - mountPath: "/root/.m2/repository"
      name: "volume-0"
      subPath: maven-repo-us-west-2-dev
      readOnly: false
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: nodejs
    image: node:13.14.0
    imagePullPolicy: IfNotPresent
    command: ['cat']
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: dind
    image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/docker-curl:latest
    imagePullPolicy: IfNotPresent
    command: ['cat']
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: aws-cli
    image: public.ecr.aws/aws-cli/aws-cli:latest
    imagePullPolicy: IfNotPresent
    command: ['cat']
    tty: true
  restartPolicy: "Never"
  volumes:
  - name: "volume-0"
    emptyDir: {}
  #  nfs:
  #    path: "/mnt/nfs/maven-repo-singapore-dev"
  #    readOnly: false
  #    server: "10.2.0.1"
  - name: "workspace-volume"
    emptyDir:
      medium: ""
'''
        }
    }
    stages {
        stage('Set Build Name') {
            steps {
              script {
                  currentBuild.displayName = "#${BUILD_NUMBER} ${BRANCH} by ${env.BUILD_USER}"
              }
            }
        }
        stage('Checkout Git') {
            steps {
              checkout([$class: 'GitSCM', branches: [[name: "${BRANCH}"]], userRemoteConfigs: [[credentialsId: '87d2e959-5740-4114-b9c7-0c42c8492cca', url: "${GIT_ADDR}"]]])
              script {
                COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
                TAG = BUILD_NUMBER + '_' + ENV + '_' + COMMIT_ID
                println "Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"
              }
            }
        }
        stage('Build a Maven project') {
            steps {
              container('nodejs') {
                  sh 'echo "nameserver 8.8.8.8" >> /etc/resolv.conf'
                  sh """
                    npm install --unsafe-perm 
                    npm run build:seasTest
                    ls dist/*
                  """
              }
          }
        }
        stage('Build a Docker Image') {
            steps {
              container('aws-cli') {
                sh """
                  export AWS_ACCESS_KEY_ID=000111
                  export AWS_SECRET_ACCESS_KEY=000111
                  if ! aws ecr describe-repositories --repository-names overseas/${JOB_BASE_NAME} &>/dev/null; then
                      echo Create repository
                      aws ecr create-repository --repository-name overseas/${JOB_BASE_NAME}
                  fi
                """
              }
              container('dind') {
                  sh '''
                    wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login
                    chmod +x docker-credential-ecr-login
                    mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login

                    mkdir -pv ~/.docker/
                    cat > ~/.docker/config.json <<EOF
{
    "credHelpers": {
        "public.ecr.aws": "ecr-login",
        "111112222222.dkr.ecr.us-west-2.amazonaws.com": "ecr-login"
    }
}
EOF

                    mkdir -pv ~/.aws/
                    cat <<'EOF'> ~/.aws/credentials
[default]
aws_access_key_id = 000111
aws_secret_access_key = 000111
EOF
                  '''
              }
              container('dind') {
                sh """
                docker -H tcp://10.0.11.234:2375 build -t  ${imgurl}:${TAG} -f- . <<'EOF'
FROM public.ecr.aws/nginx/nginx:latest
LABEL maintainer=ci
ENV SERVICE_NAME=manage-web SERVICE_PATH=/opt/manage-web
COPY dist \${SERVICE_PATH}/site
EXPOSE 80
WORKDIR \${SERVICE_PATH}
ENV TZ Asia/Shanghai
EOF
"""
                sh "docker -H tcp://10.0.11.234:2375 tag  ${imgurl}:${TAG}  ${imgurl}:latest"
              }
              container('dind') {
                sh """
                  #docker -H tcp://10.0.11.234:2375 login --username=abc -p 8ql6,yhY registry-vpc.ap-southeast-1.aliyuncs.com
                  docker -H tcp://10.0.11.234:2375 push ${imgurl}:${TAG}
                  docker -H tcp://10.0.11.234:2375 push ${imgurl}:latest
                  docker -H tcp://10.0.11.234:2375 rmi -f ${imgurl}:${TAG} ${imgurl}:latest
                """
              }
            }
        }
        stage('Deploy Trigger') {
            agent { label 'master' }
            steps {
              sh "k8s_set_tag_aws_uswest2_dev ${JOB_BASE_NAME} ${TAG}"
            }
        }
        stage("Deploy Status"){
            agent { label 'master' }
            steps {
              sh "k8s_log_aws_uswest2_dev ${JOB_BASE_NAME}"
            }
        }
    }
}

脚本式（后端） 

def giturl = "git@codeup.git/abcd/w/a.git"
def imgurl = "1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/${JOB_BASE_NAME}"
def pkg = "a/target/ap.jar"
def pom = 'pom.xml'

podTemplate(
cloud: "aws-overseas",
yaml: """

apiVersion: v1
kind: Pod
spec:
  #imagePullSecrets:
  #- name: abcd-hub
  containers:
  - name: jnlp
    image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/jnlp-slave:latest
    imagePullPolicy: IfNotPresent
    args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)']
    workingDir: /home/jenkins/agent
    tty: true
    volumeMounts:
    - mountPath: "/root/.m2/repository"
      name: "volume-0"
      subPath: maven-repo-us-west-2-dev
      readOnly: false
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: maven
    image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/maven:3.6-jdk-8-v3
    imagePullPolicy: IfNotPresent
    workingDir: /home/jenkins/agent
    command: ['cat']
    tty: true
    volumeMounts:
    - mountPath: "/root/.m2/repository"
      name: "volume-0"
      subPath: maven-repo-us-west-2-dev
      readOnly: false
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: dind
    image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/docker-curl:latest
    imagePullPolicy: IfNotPresent
    command: ['cat']
    tty: true
    volumeMounts:
    - mountPath: "/root/.m2/repository"
      name: "volume-0"
      subPath: maven-repo-us-west-2-dev
      readOnly: false
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
    workingDir: "/home/jenkins/agent"
  - name: aws-cli
    image: public.ecr.aws/aws-cli/aws-cli:latest
    imagePullPolicy: IfNotPresent
    command: ['cat']
    tty: true
  restartPolicy: "Never"
  volumes:
  - name: "volume-0"
    #emptyDir: {}
    persistentVolumeClaim:
      claimName: efs-claim-maven
  #  nfs:
  #    path: "/mnt/nfs/maven-repo-singapore-dev"
  #    readOnly: false
  #    server: "10.0.11.234"
  - name: "workspace-volume"
    emptyDir:
      medium: ""
"""
  ) {

  node(POD_LABEL) {
    stage('Set Build Name') {
        script {
            currentBuild.displayName = "#${BUILD_NUMBER} ${branch} by ${env.BUILD_USER}"
        }
    }
    stage('Checkout Git') {
        checkout([$class: 'GitSCM', branches: [[name: '${branch}']], extensions: [[$class: 'CloneOption', shallow: true]],userRemoteConfigs: [[credentialsId: '87d2e959-5740-4114-b9c7-0c42c8492cca', url: giturl]]])
    }
    stage('Build a Maven project') {
      container('maven') {
          sh 'echo "nameserver 8.8.8.8" >> /etc/resolv.conf'
          sh 'mvn -B -Dmaven.test.skip=true -P overseasdev clean package -f ' + pom
      }
      container('aws-cli') {
              sh """
                export AWS_ACCESS_KEY_ID=000111
                export AWS_SECRET_ACCESS_KEY=000111
                if ! aws ecr describe-repositories --repository-names overseas/${JOB_BASE_NAME} &>/dev/null; then
                    echo Create repository
                    aws ecr create-repository --repository-name overseas/${JOB_BASE_NAME}
                fi
              """
      }
    } 
    container('dind') {
        stage('Build a Docker Image') {
            sh '''
              wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login
              chmod +x docker-credential-ecr-login
              mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login

              mkdir -pv ~/.docker/
              cat > ~/.docker/config.json <<EOF
{
    "credHelpers": {
        "public.ecr.aws": "ecr-login",
        "1111222.dkr.ecr.us-west-2.amazonaws.com": "ecr-login"
    }
}
EOF

              mkdir -pv ~/.aws/
              cat <<'EOF'> ~/.aws/credentials
[default]
aws_access_key_id = 000111
aws_secret_access_key = 000111
EOF
            '''
            
            sh 'docker -H tcp://10.0.11.234:2375 build -t ' + imgurl + ':${BUILD_NUMBER} -f- .' + """<<EOF
FROM 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/openjdk:8-jdk-arthas-start-v8-dev
COPY """ + pkg + """ /ROOT.jar
WORKDIR /
ENV TZ Asia/Shanghai
ENTRYPOINT ["java","-jar"]
EOF

"""
            sh 'docker -H tcp://10.0.11.234:2375 tag ' + imgurl + ':${BUILD_NUMBER} ' + imgurl + ':latest'
        }
        stage('Push Docker Image') {
            //sh 'docker -H tcp://10.0.11.234:2375 login --username=abc -p 8ql6,yhY 1111222.dkr.ecr.us-west-2.amazonaws.com'
            sh 'docker -H tcp://10.0.11.234:2375 push ' + imgurl + ':${BUILD_NUMBER}'
            sh 'docker -H tcp://10.0.11.234:2375 push ' + imgurl + ':latest'
            sh 'docker -H tcp://10.0.11.234:2375 rmi -f ' + imgurl + ':${BUILD_NUMBER} ' + imgurl + ':latest'
        }
    }
  }
}
node{
    stage('Deploy Trigger') {
        sh 'k8s_set_tag_aws_uswest2_dev ${JOB_BASE_NAME} ${BUILD_NUMBER}'
        }
    stage("Deploy Status"){
        sh 'k8s_log_aws_uswest2_dev ${JOB_BASE_NAME}'
    }
}

cat <<\EOF> /usr/bin/k8s_set_tag_aws_uswest2_dev
job_name=$1
build_num=$2
repo_url=111222.dkr.ecr.us-west-2.amazonaws.com/overseas/
img_url=$repo_url$job_name

kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev  -n dev1 get deploy ${job_name}
if [ $? -eq 0 ]; then
    kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev  -n dev1 set image deploy ${job_name} ${job_name}=$img_url:$build_num
else
    ## 新建资源 端口随机，不好处理。
    kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev  -n dev1 set image deploy ${job_name} ${job_name}=$img_url:$build_num
fi
EOF

cat <<\EOF> /usr/bin/k8s_log_aws_uswest2_dev
job_name=$1
config="-n dev1 --kubeconfig /mnt/jenkins/config_aws_uswest2_dev"
pod_name=$(kubectl ${config} get pod --sort-by=.metadata.creationTimestamp -l app=${job_name}|tail -1|awk '{print $1}')
# 检测pod启动后再获取log
while true; do
kubectl ${config} get pod $pod_name 2>/dev/null |grep -q Running 
  if [ $? -eq 0 ];then
      echo $pod_name "is running"
      break 
  fi
  echo $pod_name "waiting to running"
  sleep 1
done
kubectl ${config} logs -f $pod_name &
#jenkins无需自己解决子进程问题
#pid=$!
while true; do
  restarts=$(kubectl $config get pod $pod_name 2>/dev/null|grep -v NAME|awk '{print $4}')
  if [ $restarts -gt 0 ];then
      #kill $pid
      exit 1
  fi
  kubectl $config rollout status deployment $job_name --watch=false 2>/dev/null|grep -q successfully
  if [ $? -eq 0 ];then
      #kill $pid
      exit 0
  fi
  sleep 1
done
EOF

jenkins-cd-and-argocd-cd

jenkins ci + argocd cd + lark 通知 

def V_GIT_URL = 'git@gitlab.xx.com:business/gateway.git'

pipeline {
    agent any
    tools {
        jdk 'jdk 17'
        maven 'maven 3.9'
    }
    options {
        withBuildUser()
    }
    environment {
        BUSINESS = "rl"
        DEPLOY_ENV = "dev"
        
        jobname = "${JOB_BASE_NAME}"
        OPS_GIT_URL = 'git@gitlab.xx.com:gitops/ops-deploy.git'
        OPS_PATH = "ops-deploy/${BUSINESS}/${jobname}/overlays/${DEPLOY_ENV}"
        image_url="public.ecr.aws/xxx/${jobname}:v${BUILD_NUMBER}"

        ARGO_APP = "${jobname}" + "-${DEPLOY_ENV}"
        ARGOCD_SERVER   = 'argocd.xxx.com:80'
        GITOPS_SHA = 'HEAD'
    }


    stages {
       stage('删除工作目录') {
            steps {
                // deleteDir()
                echo 'no delete'
           }
        }
        
        stage('拉取代码') {
            when {
                not {
                    environment name: 'GIT_BRANCH', value: ''
                }
            }
            options {
                retry(2)
                // timeout(time: 30, unit: 'SECONDS')
            }
            steps {
                script {
                    currentBuild.displayName = "#${BUILD_NUMBER} ${GIT_BRANCH} by ${env.BUILD_USER}"

                    echo "开始拉取 ${GIT_BRANCH} 分支代码"
                    // sh "git clone -b ${GIT_BRANCH} ${V_GIT_URL} ."
                    git branch: env.GIT_BRANCH, credentialsId: 'git-ci', url: V_GIT_URL
                }
            }
        }

        stage('mvn编译构建') {
            steps {
                script {
                    echo "开始打包api-doc应用"
                    sh "mvn clean install"
                }
            }
        }

        stage('登录AWS ECR Public') {
            steps {
                script {
                    sh """
                        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/xxx
                    """
                }
            }
        }
        
        stage('构建Docker镜像及推送镜像') {
            steps {
                script {
                    sh """
                    cat > Dockerfile << 'EOF'
FROM bellsoft/liberica-openjdk-rocky:17.0.16-cds
WORKDIR /app
COPY ./target/gateway.jar /app/app.jar
CMD ["java", "-jar", "app.jar", "--spring.profiles.active=test"]
EXPOSE 8081
EOF
                    ls -al
                    docker build -t ${image_url} .
                    docker push ${image_url}
                    """
                }
            }
        }

        stage('生成部署deployment清单') {
            steps {
                script {
                  //configFileProvider([configFile(fileId: 'eks-config', targetLocation: "eks-config")]){
                  //    sh """
                  //    /root/bin/kubectl --kubeconfig=/root/.kube/config-dev set image deployment/${jobname} ${jobname}=${image_url}
                  //    """
                  //} 
                  dir("ops-deploy") {
                    // 不显示提交历史
                    git branch: 'main',  changelog: false, poll: false, credentialsId: 'git-argocd', url: OPS_GIT_URL
                  }
                  dir(OPS_PATH) {
                    withCredentials([sshUserPrivateKey(credentialsId: 'git-argocd', keyFileVariable: 'PK')]) {
                      sh """
                      ls -l
                      yq '.images[0].newTag = "v" + strenv(BUILD_NUMBER)' kustomization.yaml 
                      git status
                      yq -i '.images[0].newTag = "v" + strenv(BUILD_NUMBER)' kustomization.yaml
                      git status
                      GIT_SSH_COMMAND="ssh -i $PK" git pull origin main
                      git add kustomization.yaml
                      git commit -m "Ci:update: ${BUSINESS}-Dev ${jobname} image: ${image_url}"
                      GIT_SSH_COMMAND="ssh -i $PK" git push  origin HEAD:main
                      """
                    }
                  }
                }
            }
        }

        stage('Argo CD Deploy') {
            steps {
                withCredentials([
                    string(
                        credentialsId: 'argocd-token',
                        variable: 'ARGOCD_AUTH_TOKEN'
                    )
                ]) {
                    sh '''
                        echo "$ARGOCD_AUTH_TOKEN"
                        argocd app sync "$ARGO_APP" \
                          --server "$ARGOCD_SERVER" \
                          --grpc-web \
                          --plaintext \
                          --auth-token "$ARGOCD_AUTH_TOKEN" \
                          --revision "$GITOPS_SHA" \
                          --timeout 600

                        argocd app wait "$ARGO_APP" \
                          --server "$ARGOCD_SERVER" \
                          --grpc-web \
                          --plaintext \
                          --auth-token "$ARGOCD_AUTH_TOKEN" \
                          --sync \
                          --health \
                          --operation \
                          --timeout 600
                    '''
                }
            }
        }

    }
    post {
        success {
            script {
                notifyLark('SUCCESS')
            }
        }

        failure {
            script {
                notifyLark('FAILURE')
            }
        }

        aborted {
            script {
                notifyLark('ABORTED')
            }
        }
    }
}

import groovy.json.JsonOutput
def notifyLark(String result) {
    def success = result == 'SUCCESS'
    def title = success ? '✅ Argo CD 发布成功' : '❌ Argo CD 发布失败'
    def color = success ? 'green' : 'red'

    withCredentials([
        string(
            credentialsId: 'lark-release-webhook',
            variable: 'LARK_WEBHOOK'
        )
    ]) {
        def payload = JsonOutput.toJson([
            msg_type: 'interactive',
            card: [
                header: [
                    template: color,
                    title: [
                        tag    : 'plain_text',
                        content: title
                    ]
                ],
                elements: [
                    [
                        tag: 'markdown',
                        content: """\
**项目：** ${BUSINESS ?: 'unknown'}
**应用：** ${ARGO_APP ?: 'unknown'}
**环境：** ${DEPLOY_ENV ?: 'unknown'}
**GitOps SHA：**  `${env.GITOPS_SHA ?: 'unknown'}`
**构建任务：** ${env.JOB_NAME} #${env.BUILD_NUMBER} by ${env.BUILD_USER}
**发布结果：** ${result}
**详情：** [查看 Jenkins 构建](${env.BUILD_URL})
""".stripIndent()
                    ]
                ]
            ]
        ])

        writeFile file: 'lark-message.json', text: payload

        sh '''
            curl --fail-with-body \
              --silent \
              --show-error \
              --request POST \
              --header 'Content-Type: application/json' \
              --data @lark-message.json \
              "$LARK_WEBHOOK"
        '''
    }
}

回溯脚本

pipeline {
    agent any
    options {
        withBuildUser()
        timestamps()
    }
    parameters {
        string(name: 'BUSINESS', defaultValue: "rl", description: '项目名称，例如 rl', trim: true)
        string(name: 'DEPLOY_ENV', defaultValue: "dev", description: '环境，例如 dev, prod', trim: true)
        string(name: 'SERVICE_NAME', defaultValue: "pc-web", description: '服务名称，例如 pc-web', trim: true)
        string(name: 'ROLLBACK_TAG', defaultValue: "v32", description: '要回滚的目标镜像TAG，例如 v32', trim: true)
        booleanParam(name: 'SKIP_ECR_CHECK', defaultValue: false, description: '跳过ECR Public镜像存在校验（应急勾选）')
        booleanParam(name: 'SKIP_ARGO_SYNC', defaultValue: false, description: '仅提交Git，不触发ArgoCD同步（调试用）')
    }
    environment {
        AWS_CREDS = credentials('aws-ecr-cred')
        GIT_USER = "rollback-ci"
        GIT_EMAIL = "rollback-ci@xxx.com"
        OPS_GIT_URL = 'git@gitlab.xx.com:gitops/ops-deploy.git'

        ARGO_APP = "${SERVICE_NAME}-${DEPLOY_ENV}"
        ARGOCD_SERVER   = 'argocd.xxx.com:80'
        GITOPS_SHA = 'HEAD'

        KUSTOMIZE_FILE = "./${BUSINESS}/${SERVICE_NAME}/overlays/${DEPLOY_ENV}/kustomization.yaml"
        ECR_PUBLIC_REPO_URI = "public.ecr.aws/xxx/${SERVICE_NAME}"
        GIT_COMMIT_MSG = "rollback ${params.SERVICE_NAME} to image tag ${ROLLBACK_TAG}"
    }
    stages {
        stage('0. 输入合法性校验') {
            steps {
                script { currentBuild.displayName = "#${BUILD_NUMBER} ${SERVICE_NAME}:ROLLBACK_TAG by ${env.BUILD_USER}" }

                sh '''
                if [ -z "${SERVICE_NAME}" ] || [ -z "${ROLLBACK_TAG}" ] || [ -z "${BUSINESS}" ] || [ -z "${DEPLOY_ENV}" ]; then
                    echo "❌ SERVICE_NAME / ROLLBACK_TAG / BUSINESS / DEPLOY_ENV 不能为空"
                    exit 1
                fi
                echo """✅ 参数校验通过
服务: ${SERVICE_NAME}
目标回滚TAG: ${ROLLBACK_TAG}
ECR Public仓库: ${ECR_PUBLIC_REPO_URI}
Kustomize文件: ${KUSTOMIZE_FILE}
                """
                '''
            }
        }
        stage('1. 镜像 Tag 存在校验') {
            when {
                expression { !params.SKIP_ECR_CHECK }
            }
            steps {
                withCredentials([string(credentialsId: 'aws-ecr-cred', variable: 'AWS_CREDS')]) {
                    sh '''
                    export AWS_ACCESS_KEY_ID=$(echo ${AWS_CREDS} | cut -d';' -f1)
                    export AWS_SECRET_ACCESS_KEY=$(echo ${AWS_CREDS} | cut -d';' -f2)
                    # ECR Public region固定 us-east-1
                    ECR_PUBLIC_REPO_NAME=${ECR_PUBLIC_REPO_URI#public.ecr.aws/}
                    ECR_PUBLIC_REPO_NAME=${ECR_PUBLIC_REPO_NAME#*/}
                    echo "检查 ECR Public 镜像 ${ECR_PUBLIC_REPO_URI}:${ROLLBACK_TAG}"
                    aws ecr-public describe-images \
                        --region us-east-1 \
                        --repository-name ${ECR_PUBLIC_REPO_NAME} \
                        --image-ids imageTag=${ROLLBACK_TAG}
                    if [ $? -eq 0 ]; then
                        echo "✅ ECR Public镜像tag ${ROLLBACK_TAG} 存在"
                    else
                        echo "❌ ECR Public镜像tag ${ROLLBACK_TAG} 不存在！终止回滚"
                        exit 1
                    fi
                    '''
                }
            }
        }
        stage('2. Checkout 部署仓库') {
            steps {
                git branch: 'main', credentialsId: 'git-argocd', url: OPS_GIT_URL

                sh '''
                if [ ! -f "${KUSTOMIZE_FILE}" ]; then
                    echo "❌ kustomization.yaml 文件 ${KUSTOMIZE_FILE} 不存在，请检查路径！"
                    exit 1
                fi
                '''
            }
        }
        stage('3. 更新 newTag') {
            steps {
                sh """
                # 获取当前配置里的 newTag
                CURRENT_TAG=\$(yq eval '.images[] | select(.name == strenv(ECR_PUBLIC_REPO_URI)) | .newTag' ${KUSTOMIZE_FILE})
                echo "当前kustomize内newTag: \${CURRENT_TAG}"
                echo "目标回滚TAG: ${ROLLBACK_TAG}"
                # 备份原文件
                cp ${KUSTOMIZE_FILE} ${KUSTOMIZE_FILE}.bak
                # yq v4 修改匹配 name 的 images 条目 newTag
                yq eval '.images[] |= (select(.name == strenv(ECR_PUBLIC_REPO_URI)) | .newTag = strenv(ROLLBACK_TAG))' -i ${KUSTOMIZE_FILE}
                echo "===== 修改前后对比 =====
旧 newTag: \${CURRENT_TAG}
新 newTag: \$(yq eval '.images[] | select(.name == strenv(ECR_PUBLIC_REPO_URI)) | .newTag' ${KUSTOMIZE_FILE})
========================"
                # yaml语法校验
                yq eval ${KUSTOMIZE_FILE} > /dev/null
                rm -f ${KUSTOMIZE_FILE}.bak
                """
            }
        }
        stage('4. 人工确认回滚操作') {
            steps {
                script {
                    def currentTag = sh(
                        script: """
                        yq eval '.images[] | select(.name == strenv(ECR_PUBLIC_REPO_URI)) | .newTag' ${KUSTOMIZE_FILE}
                        """,
                        returnStdout: true
                    ).trim() // 去掉末尾换行符

                    input message: """
⚠️ 回滚确认（Kustomize + ECR Public）
> 🔒 仅 jasper / abc / def 用户可以审批本次回滚！

服务：${SERVICE_NAME}
当前配置镜像TAG：${currentTag}
目标回滚镜像TAG：${ROLLBACK_TAG}
ECR Public仓库：${ECR_PUBLIC_REPO_URI}
Kustomize文件：${KUSTOMIZE_FILE}
ArgoCD App：${ARGO_APP}

确认提交Git并执行回滚吗？
""",
                    ok: "确认执行回滚",
                    submitter: 'jasper,bx,lemon'
                }
            }
        }
        stage('5. Git提交并推送ops-deploy') {
            steps {
                script {
                    withCredentials([sshUserPrivateKey(credentialsId: 'git-argocd', keyFileVariable: 'PK')]) {
                      sh """
                      ls -l
                      git status
                      git config user.name ${GIT_USER}
                      git config user.email ${GIT_EMAIL}
                      GIT_SSH_COMMAND="ssh -i $PK" git pull origin main
                      git add ${KUSTOMIZE_FILE}
                      git commit -m "${GIT_COMMIT_MSG}"
                      GIT_SSH_COMMAND="ssh -i $PK" git push origin HEAD:main
                      echo "✅ Git推送完成"
                      """
                    }
                }
            }
        }
        stage('6. ArgoCD 同步并等待健康') {
            steps {
                withCredentials([
                    string(
                        credentialsId: 'argocd-token',
                        variable: 'ARGOCD_AUTH_TOKEN'
                    )
                ]) {
                    sh '''
                        echo "$ARGOCD_AUTH_TOKEN"
                        argocd app sync "$ARGO_APP" \
                          --server "$ARGOCD_SERVER" \
                          --grpc-web \
                          --plaintext \
                          --auth-token "$ARGOCD_AUTH_TOKEN" \
                          --revision "$GITOPS_SHA" \
                          --timeout 300

                        argocd app wait "$ARGO_APP" \
                          --server "$ARGOCD_SERVER" \
                          --grpc-web \
                          --plaintext \
                          --auth-token "$ARGOCD_AUTH_TOKEN" \
                          --sync \
                          --health \
                          --operation \
                          --timeout 300

                        echo "✅ ArgoCD同步完成，应用健康"
                    '''
                }
            }
        }

    }
    post {
        success {
            echo "🎉 回滚任务执行成功！服务 ${SERVICE_NAME} 已回滚至镜像tag ${ROLLBACK_TAG}"
            script { notifyLark('SUCCESS') }
        }
        failure {
            echo "❌ 回滚任务失败，请检查日志！服务 ${SERVICE_NAME}"
            script { notifyLark('FAILURE') }
        }
        aborted {
            echo "⚠️ 回滚任务被取消或者非授权用户拒绝审批"
            script { notifyLark('ABORTED') }
        }
        always {
            cleanWs()
        }
    }
}

import groovy.json.JsonOutput
def notifyLark(String result) {
    def success = result == 'SUCCESS'
    def title = success ? '✅ Argo CD 回滚成功' : '❌ Argo CD 回滚失败'
    def color = success ? 'green' : 'red'

    withCredentials([
        string(
            credentialsId: 'lark-release-webhook',
            variable: 'LARK_WEBHOOK'
        )
    ]) {
        def payload = JsonOutput.toJson([
            msg_type: 'interactive',
            card: [
                header: [
                    template: color,
                    title: [
                        tag    : 'plain_text',
                        content: title
                    ]
                ],
                elements: [
                    [
                        tag: 'markdown',
                        content: """\
**服   务：** ${SERVICE_NAME}.${DEPLOY_ENV ?: 'unknown'}.${BUSINESS ?: 'unknown'}
**回滚TAG：** ${ROLLBACK_TAG ?: 'unknown'}
**构建任务：** ${env.JOB_NAME} #${env.BUILD_NUMBER} by ${env.BUILD_USER}
**发布结果：** ${result}
**详情：** [查看 Jenkins 构建](${env.BUILD_URL})
""".stripIndent()
                    ]
                ]
            ]
        ])

        writeFile file: 'lark-message.json', text: payload

        sh '''
            curl --fail-with-body \
              --silent \
              --show-error \
              --request POST \
              --header 'Content-Type: application/json' \
              --data @lark-message.json \
              "$LARK_WEBHOOK"
        '''
    }
}














服务发布更新减少耗时方案





maven构建速度： https://springdoc.cn/maven-fast-build/#google_vignette






构建工具优化-maven





0.构建耗时





pom.xml





<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-profiler-plugin</artifactId>
            <version>1.7</version>
        </plugin>
    </plugins>
</build>







执行





#cdp-server
mvn -B -Dmaven.test.skip=true -Dprofile -P <env> clean package -f pom.xml
#cat profiler
cat .profiler/xxx.html







构建耗时：1m09s





1.参数优化(无效果)





#export MAVEN_OPTS=-Xms4096m -Xmx4096m
export MAVEN_OPTS='-XX:-TieredCompilation -XX:TieredStopAtLevel=1'
mvn -T 2C -B -Dmaven.test.skip=true -P <xxx> clean package -f pom.xml






    

  • export MAVEN_OPTS='-XX:-TieredCompilation -XX:TieredStopAtLevel=1' 禁用默认分层编译
    • 

  • -B 以批处理(batch)模式运行
    • 

  • -T 2C 每一个CPU核使用2个线程，逻辑CPU核数. 模块化编译效果更佳 -pl module1,module2 -am
    • 

  • -Dmaven.test.skip = true 跳过单元测试。使用 -DskipTests 选项跳过测试执行，但仍会编译 test 文件夹
    • 

  • -Dmaven.compile.fork=true 指明多线程进行编译
    • 

  • clean 通常情况下，我们建议用户在构建时使用 clean 参数保证构建的正确性。clean 可以删除旧的构建产物，但其实我们大多数时间可能不需要这个参数，只有在某些情况下（比如，更改了类名，或者删除了一些类）才必须使用这个参数，所以，如果某次变更只是修改了一些方法，或者增加了一些类，那么就不需要强制执行 clean 了
    • 





参数选项参考：






2.maven子项目mvnd构建（无效果）





wget https://downloads.apache.org/maven/mvnd/1.0.1/maven-mvnd-1.0.1-linux-amd64.tar.gz

export PATH=/home/jenkins/agent/maven-mvnd-1.0.1-linux-amd64/bin:$PATH
#构建
mvnd -T 2C -B -Dmaven.test.skip=true -P <xxx> clean package -f pom.xml







3.增量编译(无效果)





参考：https://stackoverflow.com/questions/8918165/does-maven-support-incremental-builds





4.缩小jar包体积





结论：不推荐。 不好实时跟进各组件依赖关系，维护成本高





参考文档：







deployment 滚动更新






shrink='
spec:
  strategy:
    rollingUpdate:
      maxSurge: 20%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: type
                    operator: NotIn
                    values:
                      - virtual-kubelet
'

expand='
spec:
  strategy:
    rollingUpdate:
      maxSurge: 100%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: type
                    operator: In
                    values:
                      - virtual-kubelet
'

kubectl patch  deploy server-tmp-eci -o yaml --dry-run='client' --type merge --patch-file <(echo "$expand")


kubectl patch  deploy server-tmp-eci --type merge --patch-file <(echo "$expand")

kubectl patch  deploy server-tmp-eci --type merge --patch-file <(echo "$shrink")