kubevirt入门

1. 前置条件

1.1 配置本地存储

# local path-provisioner # https://github.com/rancher/local-path-provisioner
wget https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.29/deploy/local-path-storage.yaml
sed -i 's#/opt/local-path-provisioner#/opt/k8s-local-path-storage#g'
crictl pull dhub.kubesre.xyz/rancher/local-path-provisioner:v0.0.29
ctr -n k8s.io i tag dhub.kubesre.xyz/rancher/local-path-provisioner:v0.0.29 docker.io/rancher/local-path-provisioner:v0.0.29
k apply -f local-path-storage.yaml

1.2 开启特性门控

# 开启StatefulSetAutoDeletePVC特性门控
# 此门控在1.27已经默认启用
sudo vim /etc/kubernetes/manifests/kube-controller-manager.yaml
# 添加参数
- --feature-gates=StatefulSetAutoDeletePVC=true
# 关闭保存后kube-controller-manager会自动重新加载

2. 安装依赖

sudo apt install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils virt-manager

3. 检查是否支持虚拟化

virt-host-validate qemu

4. 安装kubevirt

# 安装kubevirt
export RELEASE=v1.3.1
kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml
kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml
# 等待所有 KubeVirt 组件都启动
kubectl -n kubevirt wait kv kubevirt --for condition=Available

# 下载 virtctl client
wget https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/virtctl-${RELEASE}-linux-amd64
mv virtctl-${RELEASE}-linux-amd64 /usr/local/bin/virtctl
chmod +x /usr/local/bin/virtctl

# 安装CDI：用于将pvc转换为虚拟机的磁盘
export RELEASE=v1.60.3
kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$RELEASE/cdi-operator.yaml
kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$RELEASE/cdi-cr.yaml

# 启用featureGates
vim enable-feature-gate.yaml
# 写入如下内容
apiVersion: kubevirt.io/v1
kind: KubeVirt
metadata:
  name: kubevirt
  namespace: kubevirt
spec:
  configuration:
    vmRolloutStrategy: "LiveUpdate"
    developerConfiguration:
      featureGates:
        - ExpandDisks
        - HotplugVolumes
        - VMLiveUpdateFeatures
  workloadUpdateStrategy:
    workloadUpdateMethods:
    - LiveMigrate

# 应用更改
kubectl apply -f enable-feature-gate.yaml

5. 编写部署helm文件

# templates/NOTES.txt
{{ include "ubuntu.fullname" . }} is deployed!
Before start, you must wait it start successfully.
You can run `kubectl get dv | grep {{ include "ubuntu.fullname" . }}` to check if the image has been pulled.
You can run `virtctl console {{ include "ubuntu.fullname" . }}` to connect to it.
You can type ctrl+] to exit console.
The username is {{ .Values.software.user }} and password is {{ .Values.software.password }}.
{{- if .Values.software.desktop.enabled }}
Please wait the desktop environment installed, then you can run `virtctl vnc {{ include "ubuntu.fullname" . }}` to connect to it.
You can see `/var/log/cloud-init-output.log` to see the progress of installing.
{{- end }}
Have a happy coding in {{ include "ubuntu.fullname" . }}!


# templates/datavolume.yaml
apiVersion: cdi.kubevirt.io/v1beta1
kind: DataVolume
metadata:
  name: {{ include "ubuntu.fullname" . }}
spec:
  pvc:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: {{ .Values.hardware.storage }}
    storageClassName: local-path
  source:
    http:
      {{- if eq .Values.software.customImageUrl ""}}
      url: "https://cloud-images.ubuntu.com/{{ .Values.software.ubuntuVersion }}/current/{{ .Values.software.ubuntuVersion }}-server-cloudimg-amd64.img"
      {{- else }}
      url: {{ .Values.software.customImageUrl }}
      {{- end }}


# templates/secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: {{ include "ubuntu.fullname" . }}-cloudinit-nocloud
type: Opaque
stringData:
  userdata: |-
    #cloud-config
    user: "{{ .Values.software.user }}"
    password: "{{ .Values.software.password }}"
    chpasswd: { expire: False }
    ssh_pwauth: True
    {{- if .Values.software.sshConfig.enabled }}
    write_files:
      - path: /home/root/ssh/{{ .Values.software.sshConfig.idFile }}
        content: |
        {{- .Values.software.sshConfig.idFileContent | nindent 18 }}
      - path: /home/root/ssh/{{ .Values.software.sshConfig.idFile }}.pub
        content: |
        {{- .Values.software.sshConfig.idFilePubContent | nindent 18 }}
    {{- end }}
    runcmd:
      - sed -i s@/archive.ubuntu.com/@/mirrors.tuna.tsinghua.edu.cn/@g /etc/apt/sources.list
      - apt-get update
      - apt-get install zsh -y
      - wget https://gitee.com/mirrors/oh-my-zsh/raw/master/tools/install.sh
      - sed -i 's/REPO=\${REPO:-ohmyzsh\/ohmyzsh}/REPO=\${REPO:-mirrors\/oh-my-zsh}/g' install.sh
      - sed -i 's/REMOTE=\${REMOTE:-https:\/\/github.com\/\${REPO}.git}/REMOTE=\${REMOTE:-https:\/\/gitee.com\/\${REPO}.git}/g' install.sh
      - sed -i 's/USER=\${USER:-\$(id -u -n)}/USER={{ .Values.software.user }}/' install.sh
      - bash install.sh
      - rm install.sh
      - sed -i 's/ZSH_THEME="robbyrussell"/ZSH_THEME="ys"/g' /home/{{ .Values.software.user }}/.zshrc
      - sed -i 's/plugins=(git)/plugins=(git zsh-syntax-highlighting zsh-autosuggestions)/g' /home/{{ .Values.software.user }}/.zshrc
      - git clone https://gitee.com/mirrors/zsh-syntax-highlighting.git /home/{{ .Values.software.user }}/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting
      - git clone https://gitee.com/mirrors/zsh-autosuggestions.git /home/{{ .Values.software.user }}/.oh-my-zsh/custom/plugins/zsh-autosuggestions
      - echo "exec -l zsh" >> /home/{{ .Values.software.user }}/.bashrc
    {{- if .Values.software.sshConfig.enabled }}
      - cp /home/root/ssh/{{ .Values.software.sshConfig.idFile }} /home/{{ .Values.software.user }}/.ssh/{{ .Values.software.sshConfig.idFile }}
      - cp /home/root/ssh/{{ .Values.software.sshConfig.idFile }}.pub /home/{{ .Values.software.user }}/.ssh/{{ .Values.software.sshConfig.idFile }}.pub
      - rm -rf /home/root/ssh
      - chown {{ .Values.software.user }}:{{ .Values.software.user }} /home/{{ .Values.software.user }}/.ssh/*
      - chmod 0600 /home/{{ .Values.software.user }}/.ssh/*
    {{- end }}
    {{- if .Values.software.desktop.enabled }}
      {{- if .Values.software.desktop.fullInstalled }}
      - DEBIAN_FRONTEND=noninteractive apt-get install ubuntu-desktop -y
      {{- else }}
      - DEBIAN_FRONTEND=noninteractive apt-get install ubuntu-desktop-minimal -y
      {{- end }}
      - DEBIAN_FRONTEND=noninteractive apt-get install lightdm debconf-utils -y
      - DEBIAN_FRONTEND=noninteractive apt-get remove gdm3 -y
      - DEBIAN_FRONTEND=noninteractive dpkg-reconfigure lightdm
      - service lightdm start
      - cat /etc/X11/default-display-manager
    {{- end }}
      - echo "Cloud init done."


# templates/vm.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  labels:
    kubevirt.io/os: linux
  name: {{ include "ubuntu.fullname" . }}
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/domain: {{ include "ubuntu.fullname" . }}
    spec:
      domain:
        devices:
          disks:
          - disk:
              bus: virtio
            name: disk0
          - disk:
              bus: virtio
              readonly: true
            name: cloudinitdisk
        machine:
          type: ""
        resources:
          requests:
            cpu: {{ .Values.hardware.cpu }}
            memory: {{ .Values.hardware.memory }}
      terminationGracePeriodSeconds: 0
      volumes:
      - name: disk0
        persistentVolumeClaim:
          claimName: {{ include "ubuntu.fullname" . }}
      - cloudInitNoCloud:
          secretRef:
            name: {{ include "ubuntu.fullname" . }}-cloudinit-nocloud
        name: cloudinitdisk


# values.yaml
# Default values for ubuntu.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# This is configs for hardware

hardware:
  storage: 40Gi
  cpu: 4
  memory: 8096M

# This is configs for software
software:
  user: ethereal
  password: "020402"
  ubuntuVersion: jammy
  # ubuntuVersion: focal
  customImageUrl: ""
  desktop:
    enabled: false
    fullInstalled: false
  sshConfig:
    enabled: false
    idFile: id_ed25519
    idFileContent: |
      <私钥>
    idFilePubContent: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID7HfYBgf1NssgJHQzmdsMaABb+qAtHRULU3p/91sQWd ethereal@Ethereal-Desktop

6. 附加额外内容

可以附加硬盘
添加如下helm文件

# values.yaml
# Default values for addvolume.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# This is configs for hardware

hardware:
  storage: 20Gi


# templates/datavolume.yaml
apiVersion: cdi.kubevirt.io/v1beta1
kind: DataVolume
metadata:
  name: {{ include "addvolume.fullname" . }}
spec:
  pvc:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: {{ .Values.hardware.storage }}
    storageClassName: local-path
  source:
    blank: {}

# templates/NOTES.txt
{{ include "addvolume.fullname" . }} is deployed!
You can run `kubectl get dv | grep {{ include "addvolume.fullname" . }}` to check the dv status.
You can attach it to a vm by `virtctl addvolume vm_name --volume-name={{ include "addvolume.fullname" . }} --persist`

可以使用如下方法附加

helm install ubuntu-vm-2204-vm2 ~/Softwares/k8s-help-tools/vm/addvolume
virtctl addvolume ubuntu-vm-2204 --volume-name=ubuntu-vm-2204-vm2 --persist

可以使用如下方法修改cpu和内存

# 停止虚拟机
virtctl stop ubuntu-vm-2204
# 修改cpu或内存配置
vim values.yaml
# 更新
helm upgrade ubuntu-vm-2204 ~/Softwares/k8s-help-tools/vm/myubuntu --set software.ubuntuVersion=jammy --set software.desktop.enabled=true --set software.sshConfig.enabled=true

7. 特殊情况

当vm关机后开启时，网卡无法启动，原因是netplan中指定了网卡的mac地址，而mac地址在关机开机后发生改变

首先进入文件夹/etc/netplan/，发现其中文件（50-cloud-init.yaml）内容为：

# This file is generated from information provided by the datasource.  Changes
# to it will not persist across an instance reboot.  To disable cloud-init's
# network configuration capabilities, write a file
# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
# network: {config: disabled}
network:
    ethernets:
        enp1s0:
            dhcp4: true
            dhcp6: true
            match:
              macaddress: b2:6a:f3:79:c0:47
            set-name: enp1s0
    version: 2

其中指明了mac地址。

因此只需要将mac地址修改为ip a所显示的mac地址，保存后使用命令sudo netplan apply部署即可。

8. 救砖

通过管理文件即可救砖。

## 通过外带的机器挂载镜像
virt-rescue -a CentOS_8_Server.img -i
## 修改密码
virt-sysprep -d 21admin --firstboot-command ‘dpkg-reconfigure openssh-server’ --password ubuntu:password:123456 --root-password password:123456
## 工具安装
apt/yum install libguestfs-tools 
systemctl start libvirtd
## 详细信息
virt-sysprep -v -x ...