hume项目k8s的改造
首先，点击jenkins构建，会选择对应的分支（如果是自动触发会获取到提交的分支，具体看jenkinsfile流程），然后jenkins会通过k8s插件调用k8s，根据jenkinsfile中定义的模板启动一个任务pod，pod中会先拉取代码，然后执行一系列的初始化操作（init.sh、拉取依赖包、并拷贝所有文件到build-work/hume，以便于后面构建镜像），再然后会build镜像，推送到
·
hume项目k8s的改造
一、修改构建目录结构
1、在根目录下添加build-work文件夹
目录结构如下
[root@k8s-worker-01 build-work]# tree .
.
├── Dockerfile
├── hume
│ └── start.sh
└── Jenkinsfile
2、每个文件内容如下
Dockerfile
FROM ccr.ccs.tencentyun.com/xxxx/php_supervisor:kafka
USER root
ENV ZBE_PATH /biz-code
ADD hume /biz-code/hume
WORKDIR /biz-code/hume
CMD ./start.sh
Jenkinsfile
pipeline {
agent {
kubernetes {cloud 'kubernetes'
cloud 'kubernetes-test'
slaveConnectTimeout 1200
workspaceVolume hostPathWorkspaceVolume(hostPath:"/opt/workspace", readOnly: false)
yaml '''
apiVersion: v1
kind: Pod
spec:
containers:
- args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\']
image: 'jenkins/jnlp-slave:latest-jdk11'
name: jnlp
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: "/etc/localtime"
name: "localtime"
readOnly: false
- command:
- "cat"
env:
- name: "LANGUAGE"
value: "en_US:en"
- name: "LC_ALL"
value: "en_US.UTF-8"
- name: "LANG"
value: "en_US.UTF-8"
image: "ccr.ccs.tencentyun.com/xxxx/php_supervisor:kafka-k8s"
imagePullPolicy: "IfNotPresent"
name: "build"
tty: true
volumeMounts:
- mountPath: "/etc/localtime"
name: "localtime"
- mountPath: "/biz-code/hume/vendor"
name: "phpdir"
readOnly: false
- command:
- "cat"
env:
- name: "LANGUAGE"
value: "en_US:en"
- name: "LC_ALL"
value: "en_US.UTF-8"
- name: "LANG"
value: "en_US.UTF-8"
image: "registry.cn-beijing.aliyuncs.com/citools/kubectl:self-1.17"
imagePullPolicy: "IfNotPresent"
name: "kubectl"
tty: true
volumeMounts:
- mountPath: "/etc/localtime"
name: "localtime"
readOnly: false
- command:
- "cat"
env:
- name: "LANGUAGE"
value: "en_US:en"
- name: "LC_ALL"
value: "en_US.UTF-8"
- name: "LANG"
value: "en_US.UTF-8"
image: "registry.cn-beijing.aliyuncs.com/citools/docker:19.03.9-git"
imagePullPolicy: "IfNotPresent"
name: "docker"
tty: true
volumeMounts:
- mountPath: "/etc/localtime"
name: "localtime"
readOnly: false
- mountPath: "/var/run/docker.sock"
name: "dockersock"
readOnly: false
restartPolicy: "Never"
imagePullSecrets:
- name: qcloudregistrykey
nodeSelector:
build: "true"
securityContext: {}
volumes:
- hostPath:
path: "/var/run/docker.sock"
name: "dockersock"
- hostPath:
path: "/usr/share/zoneinfo/Asia/Shanghai"
name: "localtime"
- name: "cachedir"
hostPath:
path: "/opt/gopkg"
- name: "phpdir"
hostPath:
path: "/opt/phppkg"
'''
}
}
stages {
stage('Pulling Code') {
parallel {
stage('Pulling Code by Jenkins') {
when {
expression {
env.giteeBranch == null
}
}
steps {
git(changelog: true, poll: true, url:'https://gitee.com/xxxx/hume.git', branch:"${BRANCH}", credentialsId: 'gitee-mima')
script {
COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
TAG = BUILD_TAG + '-' + COMMIT_ID
println "Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"
}
}
}
stage('Pulling Code by trigger') {
when {
expression {
env.giteeBranch != null
}
}
steps {
git(url: 'https://gitee.com/xxxx/hume.git', branch: env.giteeBranch, changelog: true, poll: true, credentialsId: 'gitee-mima')
script {
COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
TAG = BUILD_TAG + '-' + COMMIT_ID
println "Current branch is ${env.giteeBranch}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"
}
}
}
}
}
stage('Building') {
steps {
container(name: 'build') {
sh """
pwd
whoami
cp Scripts/init.sh.dev init.sh
chmod +x init.sh
./init.sh
composer config -g --unset repos.packagist
composer config repo.packagist composer https://mirrors.cloud.tencent.com/composer/
composer update
sudo chmod -R 777 ./*
sudo rsync -avz --exclude build-work ./* build-work/hume/
"""
}
}
}
stage('Docker build for creating image') {
environment {
HARBOR_USER = credentials('registry-secret')
}
steps {
container(name: 'docker') {
sh """
cd build-work
echo ${HARBOR_USER_USR} ${HARBOR_USER_PSW} ${TAG}
docker login -u ${HARBOR_USER_USR} -p ${HARBOR_USER_PSW} ${HARBOR_ADDRESS}
docker build -t ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG} .
docker push ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG}
"""
}
}
}
stage('Deploying to K8s') {
environment {
MY_KUBECONFIG = credentials('k8s-config')
}
steps {
container(name: 'kubectl'){
sh """
/usr/local/bin/kubectl --kubeconfig $MY_KUBECONFIG set image deploy -l app=${IMAGE_NAME} ${IMAGE_NAME}=${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG} -n $NAMESPACE
"""
}
}
}
}
environment {
COMMIT_ID = ""
HARBOR_ADDRESS = "ccr.ccs.tencentyun.com"
REGISTRY_DIR = "xxxx"
IMAGE_NAME = "hume"
NAMESPACE = "dev"
TAG = ""
}
parameters {
gitParameter(branch: '', branchFilter: 'origin/(.*)', defaultValue: 'dev', description: 'Branch for build and deploy', name:'BRANCH', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH')
}
}
hume/start.sh
mkdir /biz-code/hume/data/logs/supervisor
supervisord -c /biz-code/hume/Config/supervisor.conf
supervisorctl restart all
二、jenkins添加流水线任务，将镜像构建流程跑通
1、jenkins界面添加pipeline，注意提前安装好对应插件
Git
Git Parameter
Git Pipeline for Blue Ocean
GitLab
Credentials
Credentials Binding
Blue Ocean
Blue Ocean Pipeline Editor
Blue Ocean Core JS
Pipeline SCM API for Blue Ocean
Dashboard for Blue Ocean
Build With Parameters
Dynamic Extended Choice Parameter Plug-In
Dynamic Parameter Plug-in
Extended Choice Parameter
List Git Branches Parameter
Pipeline
Pipeline: Declarative
Kubernetes
Kubernetes CLI
Kubernetes Credentials
Image Tag Parameter
Active Choices
说明： 第一步和第二步的一个流程说明
首先，点击jenkins构建，会选择对应的分支（如果是自动触发会获取到提交的分支，具体看jenkinsfile流程），然后jenkins会通过k8s插件调用k8s，根据jenkinsfile中定义的模板启动一个任务pod，pod中会先拉取代码，然后执行一系列的初始化操作（init.sh、拉取依赖包、并拷贝所有文件到build-work/hume，以便于后面构建镜像），再然后会build镜像，推送到镜像仓库，最后就是使用kubectl命令执行更新服务器pod的镜像版本发布
三、业务运行的deploy和service
1、业务pod是以sidecar模式运行的，一个业务container，一个日志收集container，通过共享目录的形式来收集业务存在容器里面的日志
hume-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hume
labels:
app: hume
spec:
selector:
matchLabels:
app: hume
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
# minReadySeconds: 30
template:
metadata:
labels:
app: hume
spec:
containers:
- name: filebeat
image: registry.cn-beijing.aliyuncs.com/dotbalo/filebeat:7.10.2
resources:
requests:
memory: "100Mi"
cpu: "10m"
limits:
cpu: "200m"
memory: "300Mi"
imagePullPolicy: IfNotPresent
env:
- name: podIp
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: podName
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: podNamespace
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: podDeployName
value: hume
- name: TZ
value: "Asia/Shanghai"
securityContext:
runAsUser: 0
volumeMounts:
- name: logpath
mountPath: /data/log/
- name: filebeatconf
mountPath: /usr/share/filebeat/filebeat.yml
subPath: usr/share/filebeat/filebeat.yml
- name: hume
image: ccr.ccs.tencentyun.com/xxxx/hume:jenkins-hume-dev-71-0f5bbb9a
imagePullPolicy: IfNotPresent
volumeMounts:
- name: logpath
mountPath: /biz-code/hume/data/logs/
env:
- name: TZ
value: "Asia/Shanghai"
- name: LANG
value: C.UTF-8
- name: LC_ALL
value: C.UTF-8
livenessProbe:
failureThreshold: 2
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
tcpSocket:
port: 7777
timeoutSeconds: 2
ports:
- containerPort: 7777
name: web
protocol: TCP
readinessProbe:
failureThreshold: 2
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
tcpSocket:
port: 7777
timeoutSeconds: 2
resources:
limits:
cpu: 994m
memory: 1170Mi
requests:
cpu: 300m
memory: 300Mi
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: qcloudregistrykey
restartPolicy: Always
securityContext: {}
serviceAccountName: default
volumes:
- name: logpath
emptyDir: {}
- name: filebeatconf
configMap:
name: filebeatconf
items:
- key: filebeat.yml
path: usr/share/filebeat/filebeat.yml
hume-service.yaml
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
app: hume
name: hume-service
namespace: dev
spec:
ports:
- name: hume
port: 7777
protocol: TCP
targetPort: 7777
selector:
app: hume
sessionAffinity: None
type: NodePort
status:
loadBalancer: {}
四、日志收集流程
日志收集流程：采用filebeat+kafka+logstash+es+kibana的形式来做的
filebeat容器以sidecar模式与业务容器绑定，收集日志推送到kafka，在kafka中创建topic，logstash会读取kafka中topic消费业务日志，并推送至es，然后由kibana进行展示
下面需要文件可在如下连接查找
https://github.com/cs81/k8s-infrastructure/tree/master/efk-7.10.2/filebeat
1、安装helm
wget https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz
tar xf helm-v3.1.2-linux-amd64.tar.gz
mv linux-amd64/ helm
cd helm/
cp -r helm /usr/local/bin/
helm version
添加两个仓库
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add ali-stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
2、安装kafka和zookeeper
helm pull bitnami/zookeeper
tar xf zookeeper-11.1.2.tgz
cd zookeeper/
helm install zookeeper -n logging --set auth.enabled=false --set allowAnonymousLogin=true --set persistence.enabled=false .
cd ../
helm pull bitnami/kafka
tar xf kafka-20.0.6.tgz
cd kafka/
helm install kafka -n logging --set zookeeper.enabled=false --set replicaCount=1 --set externalZookeeper.servers=zookeeper --set persistence.enabled=false .
kubectl get pod -n logging
3、安装logstash
filebeat-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeatconf
data:
filebeat.yml: |-
filebeat.inputs:
- input_type: log
paths:
- /data/log/*/*.log
tail_files: true
fields:
pod_name: '${podName}'
pod_ip: '${podIp}'
pod_deploy_name: '${podDeployName}'
pod_namespace: '${podNamespace}'
output.kafka:
hosts: ["kafka.logging:9092"]
topic: "filebeat-sidecar"
codec.json:
pretty: false
keep_alive: 30s
logstash-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-configmap
data:
logstash.yml: |
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
logstash.conf: |
# all input will come from filebeat, no local logs
input {
kafka {
enable_auto_commit => true
auto_commit_interval_ms => "1000"
bootstrap_servers => "kafka:9092"
topics => ["filebeat-sidecar"]
type => ["filebeat-sidecar"]
codec=>multiline{
pattern=>"^\d{4}-"
negate=>true
what=>"previous"
}
}
}
output {
stdout{ codec=>rubydebug}
if [type] == "filebeat-sidecar"{
elasticsearch {
hosts => ["172.16.64.12:6123"]
index => "filebeat-%{+YYYY.MM.dd}"
}
} else{
elasticsearch {
hosts => ["172.16.64.12:6123"]
index => "other-input-%{+YYYY.MM.dd}"
}
}
}
logstash-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash-deployment
spec:
selector:
matchLabels:
app: logstash
replicas: 1
template:
metadata:
labels:
app: logstash
spec:
containers:
- name: logstash
image: registry.cn-beijing.aliyuncs.com/dotbalo/logstash:7.10.1
ports:
- containerPort: 5044
volumeMounts:
- name: config-volume
mountPath: /usr/share/logstash/config
- name: logstash-pipeline-volume
mountPath: /usr/share/logstash/pipeline
volumes:
- name: config-volume
configMap:
name: logstash-configmap
items:
- key: logstash.yml
path: logstash.yml
- name: logstash-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: logstash.conf
path: logstash.conf
logstash-service.yaml
kind: Service
apiVersion: v1
metadata:
name: logstash-service
spec:
selector:
app: logstash
ports:
- protocol: TCP
port: 5044
targetPort: 5044
type: ClusterIP
然后es和kibana因为之前装过，并且使用docker安装的，这里就不写了，上面logstash配置里面注意填写对应的es地址
整个流程基本就是这样，后续再添加监控及其他
K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容
更多推荐
0
0- 0
已为社区贡献27条内容
所有评论(0)