kubernetes容器集群管理（6）- Dashboard&&Metrics Server部署

1、部署dashboard1.10.1版本不支持1.15以上k8s，ui页面出现404跳转2.0.0-rc5部署后无法发现pod，尝试本地镜像无作用2.0.0-beat4版本暂时正常，兼容性日后确认#github地址https://github.com/kubernetes/dashboard/releases#部署[root@manage01 das...

情绪零碎碎

1596人浏览 · 2020-03-06 06:10:14

情绪零碎碎 · 2020-03-06 06:10:14 发布

2、metrics-server.yaml

一、部署dashboard

1.10.1版本不支持1.15以上k8s，ui页面出现404跳转
2.0.0-rc5 部署后无法发现pod，尝试本地镜像无作用，后发现是配置新增runasroot等
2.0.0-beat4版本暂时正常，兼容性日后确认

今天发现ui页面打不开了，重装了一遍还是不行，最后确认时因为开个全局代理--#

#github地址
https://github.com/kubernetes/dashboard/releases

1、部署

[root@manage01 dashboard]# cd /opt/kubernetes/dashboard
[root@manage01 dashboard]# 貌似现在需要科学：wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml
[root@manage01 dashboard]# mv recommended.yaml  k8s-dashboard.yaml
[root@manage01 dashboard]# vim ..../dashboard/kubernetes-dashboard.yaml
kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  type: NodePort
  ports:
    - port: 443
      targetPort: 8443
      noePort: 30000
  selector:
    k8s-app: kubernetes-dashboard

2、执行，查看pod情况

[root@manage01 dashboard]# kubectl apply -f k8s-dashboard.yaml
[root@manage01 dashboard]# kubectl get pod --all-namespaces
kubernetes-dashboard   dashboard-metrics-scraper-566cddb686-zlfrk   1/1     Running       0          2m38s
kubernetes-dashboard   kubernetes-dashboard-7b5bf5d559-bbsdj        1/1     Running       0          2m44s
[root@manage01 dashboard]# kubectl get all --all-namespaces

3、创建dashboard管理员

[root@manage01 dashboard]# cat dashboard-admin.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: dashboard-admin
  namespace: kubernetes-dashboard
[root@manage01 dashboard]# kubectl create -f dashboard-admin.yaml
[root@manage01 dashboard]# kubectl describe sa dashboard-admin -n kubernetes-dashboard

4、给管理员授权

[root@manage01 dashboard]# cat dashboard-admin-bind-cluster-role.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: dashboard-admin-bind-cluster-role
  labels:
    k8s-app: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: dashboard-admin
  namespace: kubernetes-dashboard
[root@manage01 dashboard]# kubectl create -f dashboard-admin-bind-cluster-role.yaml 
kubectl describe  clusterrolebinding dashboard-admin

5、获取token页面登陆

[root@manage01 dashboard]# kubectl describe secret -n kubernetes-dashboard $(kubectl get secrets -n kubernetes-dashboard | awk '/dashboard-admin/{print $1}' )

#此时firefox浏览器访问正常，至于Google浏览器证书过期问题懒得搞了
https://<NodeIP>:30000

二、部署Metrics Server

目前访问页面左侧菜单栏Overview-pods列表
上面Dashboard的CPU Usage (cores)和Memory Usage (bytes)列是空的，这是因为Kubernetes的早期版本依靠Heapster来实现完整的性能数据采集和监控功能，Kubernetes从1.8版本开始，性能数据开始以Metrics API的方式提供标准化接口，并且从1.10版本开始将Heapster替换为Metrics Server。说白了，想要页面完善还需要部署metrics server服务。

1、部署heapster

heapster
此模块，在1.8版本以后由metricserver替代，如果想部署试试可参考

wget https://github.com/kubernetes/heapster/archive/v1.5.4.tar.gz
tar -zxf v1.5.4.tar.gz
cd heapster-1.5.4/
kubectl create -f deploy/kube-config/rbac/heapster-rbac.yaml ##授权
kubectl create -f deploy/kube-config/standalone/heapster-controller.yaml

2、配置metrics service

20211123更新：之前方法不好使了，这是最新的部署文件：

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:aggregated-metrics-reader
  labels:
    rbac.authorization.k8s.io/aggregate-to-view: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules:
- apiGroups: ["metrics.k8s.io"]
  resources: ["pods", "nodes"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: metrics-server:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: metrics-server-auth-reader
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.metrics.k8s.io
spec:
  service:
    name: metrics-server
    namespace: kube-system
  group: metrics.k8s.io
  version: v1beta1
  insecureSkipTLSVerify: true
  groupPriorityMinimum: 100
  versionPriority: 100
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      hostNetwork: true
      containers:
      - name: metrics-server
        image: registry.aliyuncs.com/google_containers/metrics-server-amd64:v0.3.6
        imagePullPolicy: IfNotPresent
        command:
          - /metrics-server
          - --metric-resolution=30s
          - --requestheader-allowed-names=aggregator
          - --kubelet-insecure-tls
          - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP
        args:
          - --cert-dir=/tmp
          - --secure-port=4443
        ports:
        - name: main-port
          containerPort: 4443
          protocol: TCP
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: false
         # runAsUser: 1000
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp
      nodeSelector:
        kubernetes.io/os: linux
        kubernetes.io/arch: "amd64"
---
apiVersion: v1
kind: Service
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    kubernetes.io/name: "Metrics-server"
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: metrics-server
  ports:
  - port: 443
    protocol: TCP
    targetPort: main-port
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:metrics-server
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - nodes
  - nodes/stats
  - namespaces
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:metrics-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system

更新结束

因为默认metrics service的镜像地址需要科学上网才能拉取，所以在创建之前，我们在node1和node2节点先准备好镜像，或者直接使用国内源也行。##docker pull bluersw/metrics-server-amd64:v0.3.6##
克隆Metrics Server GitHub仓库：
[root@manage01 dashboard]# yum install git -y
[root@manage01 dashboard]# git clone https://github.com/kubernetes-sigs/metrics-server.git
[root@manage01 dashboard]# vim metrics-server/deploy/kubernetes/metrics-server-deployment.yaml

[root@manage01 kubernetes]# cat metrics-server-deployment.yaml 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      containers:
      - name: metrics-server
##########换成国内镜像可下载，或者node节点提前pull##########
        image: htcfive/metrics-server-amd64:v0.3.6
        args:
          - --cert-dir=/tmp
          - --secure-port=4443
        ports:
        - name: main-port
          containerPort: 4443
          protocol: TCP
        securityContext:
          readOnlyRootFilesystem: true
##########修改为false，注释掉runasuser，否则启动pod报错##########
          runAsNonRoot: false
#          runAsUser: 1000
##########镜像下载方式如下，command参数添加##########
        imagePullPolicy: IfNotPresent
        command:
          - /metrics-server
          - --metric-resolution=30s 
          - --kubelet-insecure-tls 
          - --kubelet-preferred-address-types=InternalIP
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp
      nodeSelector:
        beta.kubernetes.io/os: linux
        kubernetes.io/arch: "amd64"

3、添加metrics-server证书

此时部署完metrics-server后，执行kubectl top node，出现报错：

Error from server (Forbidden): nodes.metrics.k8s.io is forbidden: 
User "system:anonymous" cannot list nodes.metrics.k8s.io at the cluster scope.

报错原因： apiserver权限的问题
解决方法： 在master节点创建metrics-server证书，并在kube-apiserver配置

3.1、metrics-server生成证书

生成证书：
cat > metrics-server-csr.json <<EOF
{
  "CN": "aggregator",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes metrics-server-csr.json | cfssljson -bare metrics-server

3.2、kube-apiserver增加配置

        [root@manage01 ssl]# cat  /opt/kubernetes/cfg/kube-apiserver
        --requestheader-allowed-names=aggregator \
        --requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \
        --requestheader-extra-headers-prefix=X-Remote-Extra- \
        --requestheader-group-headers=X-Remote-Group \
        --requestheader-username-headers=X-Remote-User \
        --proxy-client-cert-file=/opt/kubernetes/ssl/metrics-server.pem \
        --proxy-client-key-file=/opt/kubernetes/ssl/metrics-server-key.pem \
        --enable-aggregator-routing=true \

--advertise-address：apiserver 对外通告的 IP（kubernetes 服务后端节点 IP）； 
--default--toleration-seconds：设置节点异常相关的阈值； 
--max--requests-inflight：请求相关的最大阈值； 
--etcd-：访问 etcd 的证书和 etcd 服务器地址； 
--experimental-encryption-provider-config：指定用于加密 etcd 中 secret 的配置； 
--bind-address： https 监听的 IP，不能为 127.0.0.1，否则外界不能访问它的安全端口 6443； 
--secret-port：https 监听端口； 
--insecure-port=0：关闭监听 http 非安全端口(8080)； 
--tls--file：指定 apiserver 使用的证书、私钥和 CA 文件； 
--audit-：配置审计策略和审计日志文件相关的参数； 
--client-ca-file：验证 client (kue-controller-manager、kube-scheduler、kubelet、kube-proxy 等)请求所带的证书； 
--enable-bootstrap-token-auth：启用 kubelet bootstrap 的 token 认证； 
--requestheader-：kube-apiserver 的 aggregator layer 相关的配置参数，proxy-client & HPA 需要使用； 
--requestheader-client-ca-file：用于签名 --proxy-client-cert-file 和 --proxy-client-key-file 指定的证书；在启用了 metric aggregator 时使用； 
--requestheader-allowed-names：不能为空，值为逗号分割的 --proxy-client-cert-file 证书的 CN 名称，这里设置为 "aggregator"； 
--service-account-key-file：签名 ServiceAccount Token 的公钥文件，kube-controller-manager 的 --service-account-private-key-file 指定私钥文件，两者配对使用； 
--runtime-config=api/all=true： 启用所有版本的 APIs，如 autoscaling/v2alpha1； 
--authorization-mode=Node,RBAC、--anonymous-auth=false： 开启 Node 和 RBAC 授权模式，拒绝未授权的请求； 
--enable-admission-plugins：启用一些默认关闭的 plugins； 
--allow-privileged：运行执行 privileged 权限的容器； 
--apiserver-count=3：指定 apiserver 实例的数量； 
--event-ttl：指定 events 的保存时间； 
--kubelet-：如果指定，则使用 https 访问 kubelet APIs；需要为证书对应的用户(上面 kubernetes.pem 证书的用户为 kubernetes) 用户定义 RBAC 规则，否则访问 kubelet API 时提示未授权； 
--proxy-client-*：apiserver 访问 metrics-server 使用的证书； 
--service-cluster-ip-range： 指定 Service Cluster IP 地址段； 
--service-node-port-range： 指定 NodePort 的端口范围； 
kube-apiserver 的 --requestheader-allowed-names 参数需要与metric证书CN字段一致，否则后续访问 metrics 时会提示权限不足。 
如果 kube-apiserver 机器没有运行 kube-proxy，则还需要添加 --enable-aggregator-routing=true 参数

3.3、kube-controller-manager配置

kube-controller-manager增加如下参数
--horizontal-pod-autoscaler-use-rest-clients=true

3.4、安装部署验证

稍等片刻，然后执行kubectl top nodes便可以看到每个节点的CPU和内存使用率了：

kubectl create -f metrics-server/deploy/kubernetes/

[root@manage01 ssl]# kubectl top nodes
NAME              CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
192.168.192.129   77m          3%     531Mi           30%       
192.168.192.130   56m          2%     287Mi           16%

四、部署kuboard

kuboard是另一个比较好用的web插件，官方自带的有些单一，推荐尝试下这个

cd /kubernetes
kubectl apply -f https://kuboard.cn/install-script/kuboard.yaml
kubectl apply -f https://addons.kuboard.cn/metrics-server/0.3.7/metrics-server.yaml
# 查看运行状态：
kubectl get pods -l k8s.kuboard.cn/name=kuboard -n kube-system
# 获取Token登录：
echo $(kubectl -n kube-system get secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}') -o go-template='{{.data.token}}' | base64 -d)

# 访问Kuboard：
http://任意一个节点的IP地址:32567

1、kuboard.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kuboard
  namespace: kube-system
  annotations:
    k8s.kuboard.cn/displayName: kuboard
    k8s.kuboard.cn/ingress: "true"
    k8s.kuboard.cn/service: NodePort
    k8s.kuboard.cn/workload: kuboard
  labels:
    k8s.kuboard.cn/layer: monitor
    k8s.kuboard.cn/name: kuboard
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s.kuboard.cn/layer: monitor
      k8s.kuboard.cn/name: kuboard
  template:
    metadata:
      labels:
        k8s.kuboard.cn/layer: monitor
        k8s.kuboard.cn/name: kuboard
    spec:
      containers:
      - name: kuboard
        image: eipwork/kuboard:latest
        imagePullPolicy: Always
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists

---
apiVersion: v1
kind: Service
metadata:
  name: kuboard
  namespace: kube-system
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    targetPort: 80
    nodePort: 32567
  selector:
    k8s.kuboard.cn/layer: monitor
    k8s.kuboard.cn/name: kuboard

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kuboard-user
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kuboard-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kuboard-user
  namespace: kube-system

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kuboard-viewer
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kuboard-viewer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: view
subjects:
- kind: ServiceAccount
  name: kuboard-viewer
  namespace: kube-system

# ---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
#   name: kuboard
#   namespace: kube-system
#   annotations:
#     k8s.kuboard.cn/displayName: kuboard
#     k8s.kuboard.cn/workload: kuboard
#     nginx.org/websocket-services: "kuboard"
#     nginx.com/sticky-cookie-services: "serviceName=kuboard srv_id expires=1h path=/"
# spec:
#   rules:
#   - host: kuboard.yourdomain.com
#     http:
#       paths:
#       - path: /
#         backend:
#           serviceName: kuboard
#           servicePort: http

2、metrics-server.yaml

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:aggregated-metrics-reader
  labels:
    rbac.authorization.k8s.io/aggregate-to-view: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules:
- apiGroups: ["metrics.k8s.io"]
  resources: ["pods", "nodes"]
  verbs: ["get", "list", "watch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: metrics-server:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: metrics-server-auth-reader
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:metrics-server
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - nodes
  - nodes/stats
  - namespaces
  verbs:
  - get
  - list
  - watch

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:metrics-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
  
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
  name: v1beta1.metrics.k8s.io
spec:
  service:
    name: metrics-server
    namespace: kube-system
    port: 443
  group: metrics.k8s.io
  version: v1beta1
  insecureSkipTLSVerify: true
  groupPriorityMinimum: 100
  versionPriority: 100

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      hostNetwork: true
      containers:
      - name: metrics-server
        image: eipwork/metrics-server:v0.3.7
        # command:
        # - /metrics-server
        # - --kubelet-insecure-tls
        # - --kubelet-preferred-address-types=InternalIP 
        args:
          - --cert-dir=/tmp
          - --secure-port=4443
          - --kubelet-insecure-tls=true
          - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,externalDNS
        ports:
        - name: main-port
          containerPort: 4443
          protocol: TCP
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
        imagePullPolicy: Always
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp
      nodeSelector:
        beta.kubernetes.io/os: linux

---
apiVersion: v1
kind: Service
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    kubernetes.io/name: "Metrics-server"
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: metrics-server
  ports:
  - port: 443
    protocol: TCP
    targetPort: 4443

Cloudpods

开源、云原生的融合云平台

更多推荐

面向未来的 IT 基础设施管理架构——融合云（Unified IaaS）

随着数字化时代的到来，IT系统已成为人类社会正常运转不可或缺的组成部分。不远的未来，智能制造，5G和人工智能等技术将成为推动生产力发展的重要引擎，人类社会将面临前所未有的全面彻底的数字化浪潮。IT基础设施作为IT系统运行的平台和载体，是实现数字化的基石。在这场数字化浪潮中，企业必须积极拥抱云计算技术，采用符合技术发展趋势、面向未来的IT基础构架，才能在未来的竞争中赢得先机。一、云计算历经十余年

Cloudpods

Cloudpods负载均衡的功能介绍

作者:周有松今天的内容会从以下几个方面展开：负载均衡产品简介。主要介绍负载均衡作为一个云上产品，它的功能模型是怎样的，日常使用中会遇到的业务词汇负载均衡的功能与典型应用场景。这部分主要结合业务词汇，对负载均衡服务中常见的一些功能选项进行介绍，并举例介绍一些典型的应用场景最后，我们做一下总结，讨论一下负载均衡产品相比传统方式的优点一、产品简介 1. 以NGINX为例提到负载均衡，我们以

Cloudpods

使用Linux vfio将Nvidia GPU透传给QEMU虚拟机

Linux 上虚拟机 GPU 透传需要使用 vfio 的方式。主要是因为在 vfio 方式下对虚拟设备的权限和 DMA 隔离上做的更好。但是这么做也有个缺点，这个物理设备在主机和其他虚拟机都不能使用了。 qemu 直接使用物理设备本身命令行是很简单的，关键在于事先在主机上对系统、内核和物理设备的一些配置。单纯从 qemu 的命令行来看，其实和普通虚拟机启动就差了最后那个-device的选项。这