[PATCH RFC 5/7] virtio-gpu-virgl: support scanout of Metal textures

[Joelle van Dyne <j@getutm.app>]

When supported, virglrenderer will return a MTLTexture handle that can be
directly used for scanout.

Signed-off-by: Joelle van Dyne <j@getutm.app>
---
 meson.build                   |  4 ++++
 include/ui/console.h          |  2 ++
 hw/display/virtio-gpu-virgl.c | 12 +++++++++++-
 hw/display/virtio-gpu.c       |  2 ++
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/meson.build b/meson.build
index d9293294d8..05bad66376 100644
--- a/meson.build
+++ b/meson.build
@@ -832,6 +832,7 @@ version_res = []
 coref = []
 iokit = []
 pvg = not_found
+metal = not_found
 emulator_link_args = []
 midl = not_found
 widl = not_found
@@ -859,6 +860,7 @@ elif host_os == 'darwin'
   host_dsosuf = '.dylib'
   pvg = dependency('appleframeworks', modules: ['ParavirtualizedGraphics', 'Metal'],
                    required: get_option('pvg'))
+  metal = dependency('appleframeworks', modules: 'Metal', required: false)
 elif host_os == 'sunos'
   socket = [cc.find_library('socket'),
             cc.find_library('nsl'),
@@ -2591,6 +2593,7 @@ if xen.found()
     ('0' + xen_version[2]).substring(-2)
   config_host_data.set('CONFIG_XEN_CTRL_INTERFACE_VERSION', xen_ctrl_version)
 endif
+config_host_data.set('CONFIG_METAL', metal.found())
 config_host_data.set('QEMU_VERSION', '"@0@"'.format(meson.project_version()))
 config_host_data.set('QEMU_VERSION_MAJOR', meson.project_version().split('.')[0])
 config_host_data.set('QEMU_VERSION_MINOR', meson.project_version().split('.')[1])
@@ -4874,6 +4877,7 @@ summary(summary_info, bool_yn: true, section: 'Crypto')
 summary_info = {}
 if host_os == 'darwin'
   summary_info += {'Cocoa support':           cocoa}
+  summary_info += {'Metal support':           metal}
 endif
 summary_info += {'D-Bus display':     dbus_display}
 summary_info += {'SDL support':       sdl}
diff --git a/include/ui/console.h b/include/ui/console.h
index 25e45295d4..a45b524c57 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -134,12 +134,14 @@ struct QemuConsoleClass {
 typedef enum ScanoutTextureNativeType {
     SCANOUT_TEXTURE_NATIVE_TYPE_NONE,
     SCANOUT_TEXTURE_NATIVE_TYPE_D3D,
+    SCANOUT_TEXTURE_NATIVE_TYPE_METAL,
 } ScanoutTextureNativeType;
 
 typedef struct ScanoutTextureNative {
     ScanoutTextureNativeType type;
     union {
         void *d3d_tex2d;
+        void *metal_texture;
     } u;
 } ScanoutTextureNative;
 
diff --git a/hw/display/virtio-gpu-virgl.c b/hw/display/virtio-gpu-virgl.c
index 36c670f988..e091eb0c76 100644
--- a/hw/display/virtio-gpu-virgl.c
+++ b/hw/display/virtio-gpu-virgl.c
@@ -452,6 +452,13 @@ static void virgl_cmd_set_scanout(VirtIOGPU *g,
 #if VIRGL_RENDERER_RESOURCE_INFO_EXT_VERSION >= SUPPORTED_VIRGL_INFO_EXT_VERSION
         if (ext.version >= VIRGL_RENDERER_RESOURCE_INFO_EXT_VERSION) {
             switch (ext.native_type) {
+#ifdef CONFIG_METAL
+            case VIRGL_NATIVE_HANDLE_METAL_TEXTURE: {
+                native.type = SCANOUT_TEXTURE_NATIVE_TYPE_METAL;
+                native.u.metal_texture = ext.native_handle;
+                break;
+            }
+#endif
             case VIRGL_NATIVE_HANDLE_NONE:
             case VIRGL_NATIVE_HANDLE_D3D_TEX2D: {
                 /* already handled above */
@@ -1198,7 +1205,10 @@ int virtio_gpu_virgl_init(VirtIOGPU *g)
     }
 #if VIRGL_VERSION_MAJOR >= 1
     if (virtio_gpu_venus_enabled(g->parent_obj.conf)) {
-        flags |= VIRGL_RENDERER_VENUS | VIRGL_RENDERER_RENDER_SERVER;
+        flags |= VIRGL_RENDERER_VENUS;
+#ifndef CONFIG_METAL /* Metal does not support render server */
+        flags |= VIRGL_RENDERER_RENDER_SERVER;
+#endif
     }
 #endif
 
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index 43e88a4daf..9cf2c15a43 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -1485,12 +1485,14 @@ void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
     VirtIOGPU *g = VIRTIO_GPU(qdev);
 
     if (virtio_gpu_blob_enabled(g->parent_obj.conf)) {
+#ifndef CONFIG_METAL
         if (!virtio_gpu_rutabaga_enabled(g->parent_obj.conf) &&
             !virtio_gpu_virgl_enabled(g->parent_obj.conf) &&
             !virtio_gpu_have_udmabuf()) {
             error_setg(errp, "need rutabaga or udmabuf for blob resources");
             return;
         }
+#endif
 
 #ifdef VIRGL_VERSION_MAJOR
     #if VIRGL_VERSION_MAJOR < 1
-- 
2.41.0