嵌套虚拟化

虚拟机管理软件：exsi 6.7，
k8s node节点: ubuntu 18.04.5 desktop, k8s 1.18.20。

Dockerfile

进入 windows10下载目录（此目录应当只有一个iso)构建镜像

cd /media/sf_backup/dev/os/windows10-20h2

cat <<EOF | sudo tee Dockerfile
FROM scratch
ADD --chown=107:107 cn_windows_10_business_editions_version_20h2_updated_april_2021_x64_dvd_0a0fc490.iso /disk/
EOF

docker build -t win10:20h2 .

win10-c-sysprep.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: win10-configmap
data:
  autounattend.xml: |-
    <?xml version="1.0" encoding="utf-8"?>
    <unattend xmlns="urn:schemas-microsoft-com:unattend">
      <settings pass="windowsPE">
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-International-Core-WinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <SetupUILanguage>
            <UILanguage>en-US</UILanguage>
          </SetupUILanguage>
          <InputLocale>0409:00000409</InputLocale>
          <SystemLocale>en-US</SystemLocale>
          <UILanguage>en-US</UILanguage>
          <UILanguageFallback>en-US</UILanguageFallback>
          <UserLocale>en-US</UserLocale>
        </component>
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-PnpCustomizationsWinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <DriverPaths>
            <PathAndCredentials wcm:keyValue="4b29ba63" wcm:action="add">
              <Path>E:\amd64\w10</Path>
            </PathAndCredentials>
            <PathAndCredentials wcm:keyValue="25fe51ea" wcm:action="add">
              <Path>E:\viostor\w10\amd64</Path>
            </PathAndCredentials>
          </DriverPaths>
        </component>
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <DiskConfiguration>
            <Disk wcm:action="add">
              <CreatePartitions>
                <CreatePartition wcm:action="add">
                  <Order>1</Order>
                  <Type>Primary</Type>
                  <Size>100</Size>
                </CreatePartition>
                <CreatePartition wcm:action="add">
                  <Extend>true</Extend>
                  <Order>2</Order>
                  <Type>Primary</Type>
                </CreatePartition>
              </CreatePartitions>
              <ModifyPartitions>
                <ModifyPartition wcm:action="add">
                  <Format>NTFS</Format>
                  <Label>System Reserved</Label>
                  <Order>1</Order>
                  <PartitionID>1</PartitionID>
                  <TypeID>0x27</TypeID>
                </ModifyPartition>
                <ModifyPartition wcm:action="add">
                  <Format>NTFS</Format>
                  <Label>OS</Label>
                  <Letter>C</Letter>
                  <Order>2</Order>
                  <PartitionID>2</PartitionID>
                </ModifyPartition>
              </ModifyPartitions>
              <DiskID>0</DiskID>
              <WillWipeDisk>true</WillWipeDisk>
            </Disk>
          </DiskConfiguration>
          <ImageInstall>
            <OSImage>
              <InstallFrom>
                <MetaData wcm:action="add">
                  <Key>/Image/Description</Key>
                  <Value>Windows 10 Pro</Value>
                </MetaData>
              </InstallFrom>
              <InstallTo>
                <DiskID>0</DiskID>
                <PartitionID>2</PartitionID>
              </InstallTo>
            </OSImage>
          </ImageInstall>
          <UserData>
            <AcceptEula>true</AcceptEula>
            <FullName/>
            <Organization/>
            <ProductKey>
              <Key/>
            </ProductKey>
          </UserData>
        </component>
      </settings>
      <settings pass="offlineServicing">
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-LUA-Settings" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <EnableLUA>false</EnableLUA>
        </component>
      </settings>
      <settings pass="specialize">
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-International-Core" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <InputLocale>0409:00000409</InputLocale>
          <SystemLocale>en-US</SystemLocale>
          <UILanguage>en-US</UILanguage>
          <UILanguageFallback>en-US</UILanguageFallback>
          <UserLocale>en-US</UserLocale>
        </component>
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-Security-SPP-UX" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <SkipAutoActivation>true</SkipAutoActivation>
        </component>
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-SQMApi" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <CEIPEnabled>0</CEIPEnabled>
        </component>
      </settings>
      <settings pass="oobeSystem">
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <OOBE>
            <HideEULAPage>true</HideEULAPage>
            <HideOEMRegistrationScreen>true</HideOEMRegistrationScreen>
            <HideOnlineAccountScreens>true</HideOnlineAccountScreens>
            <HideWirelessSetupInOOBE>true</HideWirelessSetupInOOBE>
            <NetworkLocation>Work</NetworkLocation>
            <SkipUserOOBE>true</SkipUserOOBE>
            <SkipMachineOOBE>true</SkipMachineOOBE>
            <ProtectYourPC>3</ProtectYourPC>
          </OOBE>
          <AutoLogon>
            <Password>
              <Value>123456</Value>
              <PlainText>true</PlainText>
            </Password>
            <Enabled>true</Enabled>
            <Username>Administrator</Username>
          </AutoLogon>
          <UserAccounts>
            <AdministratorPassword>
              <Value>123456</Value>
              <PlainText>true</PlainText>
            </AdministratorPassword>
          </UserAccounts>
          <RegisteredOrganization/>
          <RegisteredOwner/>
          <TimeZone>China Standard Time</TimeZone>
          <FirstLogonCommands>
            <SynchronousCommand wcm:action="add">
              <CommandLine>powershell -ExecutionPolicy Bypass -NoExit -NoProfile f:\post-install.ps1</CommandLine>
              <RequiresUserInput>false</RequiresUserInput>
              <Order>1</Order>
              <Description>Post Installation Script</Description>
            </SynchronousCommand>
          </FirstLogonCommands>
        </component>
      </settings>
    </unattend>


  post-install.ps1: |-
    # Remove AutoLogin
    # https://docs.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-shell-setup-autologon-logoncount#logoncount-known-issue
    reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" /v AutoAdminLogon /t REG_SZ /d 0 /f

    # install Qemu Tools (Drivers)
    #Start-Process msiexec -Wait -ArgumentList '/i e:\virtio-win-gt-x64.msi /qn /passive /norestart'

    # install Guest Agent
    #Start-Process msiexec -Wait -ArgumentList '/i e:\guest-agent\qemu-ga-x86_64.msi /qn /passive /norestart'

    # Rename cached unattend.xml to avoid it is picked up by sysprep
    mv C:\Windows\Panther\unattend.xml C:\Windows\Panther\unattend.install.xml

    # Eject CD, to avoid that the autounattend.xml on the CD is picked up by sysprep
    (new-object -COM Shell.Application).NameSpace(17).ParseName('F:').InvokeVerb('Eject')

    # Run Sysprep and Shutdown
    C:\Windows\System32\Sysprep\sysprep.exe /generalize /oobe /shutdown /mode:vm
    #C:\Windows\System32\Sysprep\sysprep.exe /generalize /oobe /shutdown

---

metadata:
  name: w10-c
apiVersion: kubevirt.io/v1alpha3
kind: VirtualMachineInstance
spec:
  domain:
    clock:
      timezone: "Asia/Shanghai"
      utc: {}
      timer: {}
    cpu:
      sockets: 4
      cores: 1
      threads: 1
    features:
      acpi: {}
      apic: {}
      hyperv:
        relaxed: {}
        vapic: {}
        vpindex: {}
        spinlocks:
          spinlocks: 8191
    resources:
      requests:
        memory: 4096M
        cpu: 4
    devices:
      #blockMultiQueue: true
      inputs:
        - type: tablet
          bus: usb
          #bus: virtio
          name: tablet1
      disks:
      - name: rootdisk
        bootOrder: 1
        disk:
          bus: virtio
      - name: containerdisk
        bootOrder: 2
        cdrom:
          bus: sata
      - name: virtiocontainerdisk
        bootOrder: 3
        cdrom:
          bus: sata
      - name: sysprep
        bootOrder: 4
        cdrom:
          bus: sata
      interfaces:
        - name: default
          model: e1000
          masquerade: {}
  networks:
  - name: default
    pod: {}
  volumes:
    - name: containerdisk
      containerDisk:
        image: win10:20h2
    - name: rootdisk
      hostDisk:
        capacity: 30Gi
        path: /data/vm/win1001/disk.img
        type: DiskOrCreate
    - name: virtiocontainerdisk
      containerDisk:
        image: kubevirt/virtio-container-disk:v0.37.0
    - name: sysprep
      sysprep:
        configMap:
          name: win10-configmap

执行命令创建vmi

kubectl create -f win10-c-sysprep.yaml 

vmi running 状态后，连接vnc界面，进行安装（此unattend.xml 有些不对，需要人工选择语言）

virtctl vnc w10-c

win10-c-ok.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: win10-ok-configmap
data:
  autounattend.xml: |-
    <?xml version="1.0" encoding="utf-8"?>
    <!-- responsible for installing windows, ignored on sysprepped images -->
  unattend.xml: |-
    <?xml version="1.0" encoding="utf-8"?>
    <unattend xmlns="urn:schemas-microsoft-com:unattend">
      <settings pass="oobeSystem">
        <component xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS">
          <OOBE>
            <HideEULAPage>true</HideEULAPage>
            <HideOEMRegistrationScreen>true</HideOEMRegistrationScreen>
            <HideOnlineAccountScreens>true</HideOnlineAccountScreens>
            <HideWirelessSetupInOOBE>true</HideWirelessSetupInOOBE>
            <NetworkLocation>Work</NetworkLocation>
            <SkipUserOOBE>true</SkipUserOOBE>
            <SkipMachineOOBE>true</SkipMachineOOBE>
            <ProtectYourPC>3</ProtectYourPC>
          </OOBE>
          <AutoLogon>
            <Password>
            <Value>123456</Value>
              <PlainText>true</PlainText>
            </Password>
            <Enabled>true</Enabled>
            <Username>Administrator</Username>
          </AutoLogon>
          <UserAccounts>
            <AdministratorPassword>
               <Value>123456</Value>
               <PlainText>true</PlainText>
            </AdministratorPassword>
          </UserAccounts>
          <RegisteredOrganization>Kuebvirt</RegisteredOrganization>
          <RegisteredOwner>Kubevirt</RegisteredOwner>
          <TimeZone>Eastern Standard Time</TimeZone>
          <FirstLogonCommands>
            <SynchronousCommand wcm:action="add">
              <CommandLine>powershell -ExecutionPolicy Bypass -NoExit -WindowStyle Hidden -NoProfile f:\setup.ps1</CommandLine>
              <RequiresUserInput>false</RequiresUserInput>
              <Order>1</Order>
              <Description>Customize Script</Description>
            </SynchronousCommand>
          </FirstLogonCommands>
          <LogonCommands>
            <AsynchronousCommand wcm:action="add">
              <CommandLine>powershell -ExecutionPolicy Bypass -NoExit -WindowStyle Hidden -NoProfile f:\setup.ps1</CommandLine>
              <RequiresUserInput>false</RequiresUserInput>
              <Order>1</Order>
              <Description>Customize Script</Description>
            </AsynchronousCommand>
          </LogonCommands>
        </component>
      </settings>
    </unattend>

  setup.ps1: |-
    date >> C:\Users\Administrator\Desktop\tmp.txt
    #after install nfs client, uncomment next 1 line
    #$r = (cmd /c mount \\192.88.1.190\data s:)

---

metadata:
  name: w10-c
apiVersion: kubevirt.io/v1alpha3
kind: VirtualMachineInstance
spec:
  domain:
    clock:
      timezone: "Asia/Shanghai"
      utc: {}
      timer: {}
    cpu:
      sockets: 4
      cores: 1
      threads: 1
    features:
      acpi: {}
      apic: {}
      hyperv:
        relaxed: {}
        vapic: {}
        vpindex: {}
        spinlocks:
          spinlocks: 8191
    resources:
      requests:
        memory: 4096M
        cpu: 4
    devices:
      blockMultiQueue: true
      inputs:
        - type: tablet
          bus: usb
          #bus: virtio
          name: tablet1
      disks:
      - name: rootdisk
        bootOrder: 1
        disk:
          bus: virtio
      - name: containerdisk
        bootOrder: 2
        cdrom:
          bus: sata
      - name: virtiocontainerdisk
        bootOrder: 3
        cdrom:
          bus: sata
      - name: sysprep
        bootOrder: 4
        cdrom:
          bus: sata
      interfaces:
        - name: default
          model: e1000
          masquerade: {}
  networks:
  - name: default
    pod: {}
  volumes:
    - name: containerdisk
      containerDisk:
        image: win10:20h2
    - name: rootdisk
      hostDisk:
        capacity: 30Gi
        path: /data/vm/win1001/disk.img
        type: DiskOrCreate
    - name: virtiocontainerdisk
      containerDisk:
        image: kubevirt/virtio-container-disk:v0.37.0
    - name: sysprep
      sysprep:
        configMap:
          name: win10-ok-configmap

按照 win10-c-sysprep.yaml 安装好windows 10 后，再使用此配置重新配置操作系统，此会自动登录，并且执行登录后的启动脚本。
注意：
（1）第一次用此yaml安装，要配置系统，较慢。以后启动很快。
（2）上面的登录后启动脚本，mount nfs服务，需要在启动系统，打开windows10 nfs客户端功能后，再放开注释