Add sgx vt support

[zixi-chen]

ID: 2135575

Signed-off-by: Zixi Chen [email protected]

[zixi-chen]

Test params: use_mem = no smp = 6 mem = 61440 vcpu_maxcpus = ${smp} policy_mem = bind vm_sgx_epc_devs = sgx0 sgx1 vm_sgx_epc_memdev_sgx0 = A vm_sgx_epc_node_sgx0 = 0 vm_sgx_epc_memdev_sgx1 = B vm_sgx_epc_node_sgx1 = 1 mem_devs += "A B mem0 mem1" backend_mem_A = "memory-backend-epc" size_mem_A = 64M prealloc_A = on host-nodes_A = "0" backend_mem_B = "memory-backend-epc" size_mem_B = 64M prealloc_B = on host-nodes_B = "1" backend_mem_mem0 = "memory-backend-ram" size_mem0 = 30000M host-nodes_mem0 = "0" backend_mem_mem1 = "memory-backend-ram" size_mem1 = 31440M host-nodes_mem1 = "1" monitor_expect_nodes = 2 guest_numa_nodes = "node0 node1" numa_nodeid_node0 = 0 numa_nodeid_node1 = 1 numa_memdev_node0 = mem-mem0 numa_memdev_node1 = mem-mem1 numa_cpus_node0 = "4,5" numa_cpus_node1 = "0,1,2,3"

Qemu cmdline: /usr/libexec/qemu-kvm
-S
-name 'avocado-vt-vm1'
-sandbox on
-blockdev node-name=file_ovmf_code,driver=file,filename=/usr/share/OVMF/OVMF_CODE.secboot.fd,auto-read-only=on,discard=unmap
-blockdev node-name=drive_ovmf_code,driver=raw,read-only=on,file=file_ovmf_code
-blockdev node-name=file_ovmf_vars,driver=file,filename=/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel920-64-virtio-scsi_qcow2_filesystem_VARS.fd,auto-read-only=on,discard=unmap
-blockdev node-name=drive_ovmf_vars,driver=raw,read-only=off,file=file_ovmf_vars
-machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars,sgx-epc.0.memdev=mem-A,sgx-epc.0.node=0,sgx-epc.1.memdev=mem-B,sgx-epc.1.node=1
-device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x1,chassis=1
-device pcie-pci-bridge,id=pcie-pci-bridge-0,addr=0x0,bus=pcie-root-port-0
-nodefaults
-device VGA,bus=pcie.0,addr=0x2
-m 61440 \ -object memory-backend-epc,size=64M,prealloc=on,policy=bind,host-nodes=0,id=mem-A
-object memory-backend-epc,size=64M,prealloc=on,policy=bind,host-nodes=1,id=mem-B
-object memory-backend-ram,size=30000M,policy=bind,host-nodes=0,id=mem-mem0
-object memory-backend-ram,size=31440M,policy=bind,host-nodes=1,id=mem-mem1
-smp 56,maxcpus=56,cores=28,threads=1,dies=1,sockets=2
-numa node,memdev=mem-mem0,cpus=4,cpus=5,nodeid=0
-numa node,memdev=mem-mem1,cpus=0,cpus=1,cpus=2,cpus=3,nodeid=1
-cpu 'Icelake-Server-noTSX',+kvm_pv_unhalt
-chardev socket,server=on,path=/var/tmp/avocado_6a1ss7cx/monitor-qmpmonitor1-20221111-004457-b3BTNew1,wait=off,id=qmp_id_qmpmonitor1
-mon chardev=qmp_id_qmpmonitor1,mode=control
-chardev socket,server=on,path=/var/tmp/avocado_6a1ss7cx/monitor-catch_monitor-20221111-004457-b3BTNew1,wait=off,id=qmp_id_catch_monitor
-mon chardev=qmp_id_catch_monitor,mode=control
-device pvpanic,ioport=0x505,id=idhdFfYM
-chardev socket,server=on,path=/var/tmp/avocado_6a1ss7cx/serial-serial0-20221111-004457-b3BTNew1,wait=off,id=chardev_serial0
-device isa-serial,id=serial0,chardev=chardev_serial0
-chardev socket,id=seabioslog_id_20221111-004457-b3BTNew1,path=/var/tmp/avocado_6a1ss7cx/seabios-20221111-004457-b3BTNew1,server=on,wait=off
-device isa-debugcon,chardev=seabioslog_id_20221111-004457-b3BTNew1,iobase=0x402
-device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x1.0x1,bus=pcie.0,chassis=2
-device qemu-xhci,id=usb1,bus=pcie-root-port-1,addr=0x0
-device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1
-device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x1.0x2,bus=pcie.0,chassis=3
-device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-2", "addr": "0x0"}'
-blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel9-64-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}'
-blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}'
-device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}'
-device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x1.0x3,bus=pcie.0,chassis=4
-device virtio-net-pci,mac=9a:1b:55:62:3c:eb,id=idpEbQEd,netdev=idMpV1ck,bus=pcie-root-port-3,addr=0x0
-netdev tap,id=idMpV1ck,vhost=on,vhostfd=16,fd=9
-vnc :0
-rtc base=utc,clock=host,driftfix=slew
-boot menu=off,order=cdn,once=c,strict=off
-chardev socket,id=char_vtpm_avocado-vt-vm1_tpm0,path=/root/avocado/data/avocado-vt/swtpm/avocado-vt-vm1_tpm0_swtpm.sock
-tpmdev emulator,chardev=char_vtpm_avocado-vt-vm1_tpm0,id=emulator_vtpm_avocado-vt-vm1_tpm0
-device tpm-crb,id=tpm-crb_vtpm_avocado-vt-vm1_tpm0,tpmdev=emulator_vtpm_avocado-vt-vm1_tpm0
-enable-kvm
-device pcie-root-port,id=pcie_extra_root_port_0,multifunction=on,bus=pcie.0,addr=0x3,chassis=5

[zixi-chen]

Hi @luckyh @zhencliu @YongxueHong, could you please help to review the patch? Thanks.

[zixi-chen]

Tested params: use_mem = no smp = 6 mem = 61440 vcpu_maxcpus = ${smp} policy_mem = bind mem_devs += "A B mem0 mem1" backend_mem_A = "memory-backend-epc" size_mem_A = 64M prealloc_A = on host-nodes_A = "0" backend_mem_B = "memory-backend-epc" size_mem_B = 64M prealloc_B = on host-nodes_B = "1" backend_mem_mem0 = "memory-backend-ram" size_mem0 = 30000M host-nodes_mem0 = "0" backend_mem_mem1 = "memory-backend-ram" size_mem1 = 31440M host-nodes_mem1 = "1" monitor_expect_nodes = 2 guest_numa_nodes = "node0 node1" numa_nodeid_node0 = 0 numa_nodeid_node1 = 1 numa_memdev_node0 = mem-mem0 numa_memdev_node1 = mem-mem1 numa_cpus_node0 = "4,5" numa_cpus_node1 = "0,1,2,3" vm_sgx_epc_devs = sgx0 sgx1 vm_sgx_epc_memdev_sgx0 = A vm_sgx_epc_node_sgx0 = ${numa_nodeid_node0} vm_sgx_epc_memdev_sgx1 = B vm_sgx_epc_node_sgx1 = ${numa_nodeid_node1} ... -machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars,sgx-epc.0.memdev=mem-A,sgx-epc.0.node=0,sgx-epc.1.memdev=mem-B,sgx-epc.1.node=1
-m 61440
-object memory-backend-epc,size=64M,prealloc=on,policy=bind,host-nodes=0,id=mem-A
-object memory-backend-epc,size=64M,prealloc=on,policy=bind,host-nodes=1,id=mem-B
-object memory-backend-ram,size=30000M,policy=bind,host-nodes=0,id=mem-mem0
-object memory-backend-ram,size=31440M,policy=bind,host-nodes=1,id=mem-mem1
-smp 56,maxcpus=56,cores=28,threads=1,dies=1,sockets=2
-numa node,memdev=mem-mem0,cpus=4,cpus=5,nodeid=0
-numa node,memdev=mem-mem1,cpus=0,cpus=1,cpus=2,cpus=3,nodeid=1
...

@zhencliu @YongxueHong @luckyh Could you please help to review again? Thanks!

[YongxueHong]

For the below commit, should be updated according to the code changes:

Add sgx vt support

1. Add new memory backend memory-backend-epc

2. Add qmp query-sgx and query-sgx-capabilities

3. Add machine sgx params:
   sgx-epc.{id}.memdev=mem-{ele_id},sgx-epc.{id}.node={numaid}

There is no changes refer to Add qmp query-sgx and query-sgx-capabilities

[zixi-chen]

For the below commit, should be updated according to the code changes:

Add sgx vt support

1. Add new memory backend memory-backend-epc

2. Add qmp query-sgx and query-sgx-capabilities

3. Add machine sgx params:
   sgx-epc.{id}.memdev=mem-{ele_id},sgx-epc.{id}.node={numaid}

There is no changes refer to Add qmp query-sgx and query-sgx-capabilities

Thanks Yongxue, changed the commit info

[YongxueHong]

Thanks all, let's merge it.