FFmpeg
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82  VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate;
83  VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy;
84  VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout;
85 
86 #ifdef VK_EXT_shader_long_vector
87  VkPhysicalDeviceShaderLongVectorFeaturesEXT long_vector;
88 #endif
89 
90 #ifdef VK_EXT_shader_replicated_composites
91  VkPhysicalDeviceShaderReplicatedCompositesFeaturesEXT replicated_composites;
92 #endif
93 
94 #ifdef VK_EXT_zero_initialize_device_memory
95  VkPhysicalDeviceZeroInitializeDeviceMemoryFeaturesEXT zero_initialize;
96 #endif
97 
98 #ifdef VK_KHR_shader_expect_assume
99  VkPhysicalDeviceShaderExpectAssumeFeaturesKHR expect_assume;
100 #endif
101 
102  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
103 #ifdef VK_KHR_video_maintenance2
104  VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2;
105 #endif
106 #ifdef VK_KHR_video_decode_vp9
107  VkPhysicalDeviceVideoDecodeVP9FeaturesKHR vp9_decode;
108 #endif
109 #ifdef VK_KHR_video_encode_av1
110  VkPhysicalDeviceVideoEncodeAV1FeaturesKHR av1_encode;
111 #endif
112 
113  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
114  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
115  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
116 
117 #ifdef VK_KHR_shader_relaxed_extended_instruction
118  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
119 #endif
120 } VulkanDeviceFeatures;
121 
122 typedef struct VulkanDevicePriv {
123  /**
124  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
125  */
126  AVVulkanDeviceContext p;
127 
128  /* Vulkan library and loader functions */
129  void *libvulkan;
130 
131  FFVulkanContext vkctx;
132  AVVulkanDeviceQueueFamily *compute_qf;
133  AVVulkanDeviceQueueFamily *transfer_qf;
134 
135  /* Properties */
136  VkPhysicalDeviceProperties2 props;
137  VkPhysicalDeviceMemoryProperties mprops;
138  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
139  VkPhysicalDeviceDriverProperties dprops;
140 
141  /* Opaque FD external semaphore properties */
142  VkExternalSemaphoreProperties ext_sem_props_opaque;
143 
144  /* Enabled features */
145  VulkanDeviceFeatures feats;
146 
147  /* Queues */
148  pthread_mutex_t **qf_mutex;
149  uint32_t nb_tot_qfs;
150  uint32_t img_qfs[64];
151  uint32_t nb_img_qfs;
152 
153  /* Debug callback */
154  VkDebugUtilsMessengerEXT debug_ctx;
155 
156  /* Settings */
157  int use_linear_images;
158 
159  /* Option to allocate all image planes in a single allocation */
160  int contiguous_planes;
161 
162  /* Disable multiplane images */
163  int disable_multiplane;
164 
165  /* Prefer memcpy over dynamic host pointer imports */
166  int avoid_host_import;
167 
168  /* Maximum queues */
169  int limit_queues;
170 } VulkanDevicePriv;
171 
172 typedef struct VulkanFramesPriv {
173  /**
174  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
175  */
176  AVVulkanFramesContext p;
177 
178  /* Image conversions */
179  FFVkExecPool compute_exec;
180 
181  /* Image transfers */
182  FFVkExecPool upload_exec;
183  FFVkExecPool download_exec;
184 
185  /* Temporary buffer pools */
186  AVBufferPool *tmp;
187 
188  /* Modifier info list to free at uninit */
189  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
190 
191  /* Properties for DRM modifier for each plane in the image */
192  VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5];
193 } VulkanFramesPriv;
194 
195 typedef struct AVVkFrameInternal {
196  pthread_mutex_t update_mutex;
197 
198 #if CONFIG_CUDA
199  /* Importing external memory into cuda is really expensive so we keep the
200  * memory imported all the time */
201  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
202  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
203  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
204  CUarray cu_array[AV_NUM_DATA_POINTERS];
205  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
206 #ifdef _WIN32
207  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
208  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
209 #endif
210 #endif
211 } AVVkFrameInternal;
212 
213 /* Initialize all structs in VulkanDeviceFeatures */
214 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
215 {
216  VulkanDevicePriv *p = ctx->hwctx;
217  FFVulkanContext *s = &p->vkctx;
218 
219  feats->device = (VkPhysicalDeviceFeatures2) {
220  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
221  };
222 
223  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
225  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
226  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
227  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
228  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
229 
230  FF_VK_STRUCT_EXT(s, &feats->device, &feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
231  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
232  FF_VK_STRUCT_EXT(s, &feats->device, &feats->subgroup_rotate, FF_VK_EXT_SUBGROUP_ROTATE,
233  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_ROTATE_FEATURES_KHR);
234  FF_VK_STRUCT_EXT(s, &feats->device, &feats->host_image_copy, FF_VK_EXT_HOST_IMAGE_COPY,
235  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_IMAGE_COPY_FEATURES_EXT);
236 
237 #ifdef VK_EXT_shader_long_vector
238  FF_VK_STRUCT_EXT(s, &feats->device, &feats->long_vector, FF_VK_EXT_LONG_VECTOR,
239  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_LONG_VECTOR_FEATURES_EXT);
240 #endif
241 
242 #ifdef VK_EXT_shader_replicated_composites
243  FF_VK_STRUCT_EXT(s, &feats->device, &feats->replicated_composites, FF_VK_EXT_REPLICATED_COMPOSITES,
244  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_REPLICATED_COMPOSITES_FEATURES_EXT);
245 #endif
246 
247 #ifdef VK_EXT_zero_initialize_device_memory
248  FF_VK_STRUCT_EXT(s, &feats->device, &feats->zero_initialize, FF_VK_EXT_ZERO_INITIALIZE,
249  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ZERO_INITIALIZE_DEVICE_MEMORY_FEATURES_EXT);
250 #endif
251 
252 #ifdef VK_KHR_shader_expect_assume
253  FF_VK_STRUCT_EXT(s, &feats->device, &feats->expect_assume, FF_VK_EXT_EXPECT_ASSUME,
254  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_EXPECT_ASSUME_FEATURES_KHR);
255 #endif
256 
257  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
258  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
259 #ifdef VK_KHR_video_maintenance2
260  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_2, FF_VK_EXT_VIDEO_MAINTENANCE_2,
261  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_2_FEATURES_KHR);
262 #endif
263 #ifdef VK_KHR_video_decode_vp9
264  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vp9_decode, FF_VK_EXT_VIDEO_DECODE_VP9,
265  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_DECODE_VP9_FEATURES_KHR);
266 #endif
267 #ifdef VK_KHR_video_encode_av1
268  FF_VK_STRUCT_EXT(s, &feats->device, &feats->av1_encode, FF_VK_EXT_VIDEO_ENCODE_AV1,
269  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_AV1_FEATURES_KHR);
270 #endif
271 
272  FF_VK_STRUCT_EXT(s, &feats->device, &feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
273  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
274  FF_VK_STRUCT_EXT(s, &feats->device, &feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
275  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
276  FF_VK_STRUCT_EXT(s, &feats->device, &feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
277  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
278  FF_VK_STRUCT_EXT(s, &feats->device, &feats->explicit_mem_layout, FF_VK_EXT_EXPLICIT_MEM_LAYOUT,
279  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_FEATURES_KHR);
280 
281 #ifdef VK_KHR_shader_relaxed_extended_instruction
282  FF_VK_STRUCT_EXT(s, &feats->device, &feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
283  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
284 #endif
285 }
286 
287 /* Copy all needed device features */
288 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
289 {
290 #define COPY_VAL(VAL) \
291  do { \
292  dst->VAL = src->VAL; \
293  } while (0) \
294 
295  COPY_VAL(device.features.shaderImageGatherExtended);
296  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
297  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
298  COPY_VAL(device.features.fragmentStoresAndAtomics);
299  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
300  COPY_VAL(device.features.shaderInt64);
301  COPY_VAL(device.features.shaderInt16);
302  COPY_VAL(device.features.shaderFloat64);
303  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
304  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
305 
306  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
307  COPY_VAL(vulkan_1_1.storagePushConstant16);
308  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
309  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
310 
311  COPY_VAL(vulkan_1_2.timelineSemaphore);
312  COPY_VAL(vulkan_1_2.scalarBlockLayout);
313  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
314  COPY_VAL(vulkan_1_2.hostQueryReset);
315  COPY_VAL(vulkan_1_2.storagePushConstant8);
316  COPY_VAL(vulkan_1_2.shaderInt8);
317  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
318  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
319  COPY_VAL(vulkan_1_2.shaderFloat16);
320  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
321  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
322  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
323  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
324  COPY_VAL(vulkan_1_2.vulkanMemoryModelAvailabilityVisibilityChains);
325  COPY_VAL(vulkan_1_2.uniformBufferStandardLayout);
326  COPY_VAL(vulkan_1_2.runtimeDescriptorArray);
327  COPY_VAL(vulkan_1_2.shaderSubgroupExtendedTypes);
328  COPY_VAL(vulkan_1_2.shaderUniformBufferArrayNonUniformIndexing);
329  COPY_VAL(vulkan_1_2.shaderSampledImageArrayNonUniformIndexing);
330  COPY_VAL(vulkan_1_2.shaderStorageBufferArrayNonUniformIndexing);
331  COPY_VAL(vulkan_1_2.shaderStorageImageArrayNonUniformIndexing);
332 
333  COPY_VAL(vulkan_1_3.dynamicRendering);
334  COPY_VAL(vulkan_1_3.maintenance4);
335  COPY_VAL(vulkan_1_3.synchronization2);
336  COPY_VAL(vulkan_1_3.computeFullSubgroups);
337  COPY_VAL(vulkan_1_3.subgroupSizeControl);
338  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
339  COPY_VAL(vulkan_1_3.dynamicRendering);
340 
341  COPY_VAL(timeline_semaphore.timelineSemaphore);
342  COPY_VAL(subgroup_rotate.shaderSubgroupRotate);
343  COPY_VAL(host_image_copy.hostImageCopy);
344 
345 #ifdef VK_EXT_shader_long_vector
346  COPY_VAL(long_vector.longVector);
347 #endif
348 
349 #ifdef VK_EXT_shader_replicated_composites
350  COPY_VAL(replicated_composites.shaderReplicatedComposites);
351 #endif
352 
353 #ifdef VK_EXT_zero_initialize_device_memory
354  COPY_VAL(zero_initialize.zeroInitializeDeviceMemory);
355 #endif
356 
357  COPY_VAL(video_maintenance_1.videoMaintenance1);
358 #ifdef VK_KHR_video_maintenance2
359  COPY_VAL(video_maintenance_2.videoMaintenance2);
360 #endif
361 
362 #ifdef VK_KHR_video_decode_vp9
363  COPY_VAL(vp9_decode.videoDecodeVP9);
364 #endif
365 
366 #ifdef VK_KHR_video_encode_av1
367  COPY_VAL(av1_encode.videoEncodeAV1);
368 #endif
369 
370  COPY_VAL(shader_object.shaderObject);
371 
372  COPY_VAL(cooperative_matrix.cooperativeMatrix);
373 
374  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
375  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
376 
377  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout);
378  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayoutScalarBlockLayout);
379  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout8BitAccess);
380  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout16BitAccess);
381 
382 #ifdef VK_KHR_shader_relaxed_extended_instruction
383  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
384 #endif
385 
386 #ifdef VK_KHR_shader_expect_assume
387  COPY_VAL(expect_assume.shaderExpectAssume);
388 #endif
389 
390 #undef COPY_VAL
391 }
392 
393 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
394 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
395 
396 static const struct FFVkFormatEntry {
397  VkFormat vkf;
398  enum AVPixelFormat pixfmt;
399  VkImageAspectFlags aspect;
400  int vk_planes;
401  int nb_images;
402  int nb_images_fallback;
403  const VkFormat fallback[5];
404 } vk_formats_list[] = {
405  /* Gray formats */
406  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
407  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
408  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
409  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
410  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
411  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GRAY32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_UINT } },
412  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
413 
414  /* RGB formats */
415 // { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
416  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
417  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
418  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
419  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
420  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
421 // { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
422  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
423 // { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
424 // { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
425  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
426  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
427  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
428  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
429 
430  /* Planar RGB */
431  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRP, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
432  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
433  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
434  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
435  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
436  { VK_FORMAT_R16_SFLOAT, AV_PIX_FMT_GBRPF16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16_SFLOAT } },
437  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
438 
439  /* Planar RGB + Alpha */
440  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
441  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
442  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
443  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
444  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
445  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAPF16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
446  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GBRAP32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT } },
447  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
448 
449  /* Bayer */
450  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_BAYER_RGGB16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
451 
452  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
453  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
454  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
455  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
456  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
457 
458  /* Two-plane 422 YUV at 8, 10 and 16 bits */
459  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
460  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
461  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
462  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
463 
464  /* Two-plane 444 YUV at 8, 10 and 16 bits */
465  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
466  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
467  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
468  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
469 
470  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
471  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
472  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
473  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
474  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
475  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
476  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
477  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
478  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
479  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
480  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
481  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
482  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
483 
484  /* Single plane 422 at 8, 10, 12 and 16 bits */
485  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
486  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
487  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
488  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
489  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
490 
491  /* Planar YUVA 420 at 8, 10 and 16 bits */
492  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA420P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
493  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
494  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
495 
496  /* Planar YUVA 422 at 8, 10, 12 and 16 bits */
497  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA422P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
498  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
499  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
500  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
501 
502  /* Planar YUVA 444 at 8, 10, 12 and 16 bits */
503  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA444P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
504  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
505  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
506  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
507 
508  /* Single plane 444 at 8, 10, 12 and 16 bits */
509 // { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
510  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
511  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
512 };
513 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
514 
515 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
516 {
517  for (int i = 0; i < nb_vk_formats_list; i++)
518  if (vk_formats_list[i].pixfmt == p)
519  return vk_formats_list[i].fallback;
520  return NULL;
521 }
522 
523 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
524 {
525  for (int i = 0; i < nb_vk_formats_list; i++)
526  if (vk_formats_list[i].pixfmt == p)
527  return &vk_formats_list[i];
528  return NULL;
529 }
530 
531 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
532  VkImageTiling tiling,
533  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
534  int *nb_images, /* Output number of images */
535  VkImageAspectFlags *aspect, /* Output aspect */
536  VkImageUsageFlags *supported_usage, /* Output supported usage */
537  int disable_multiplane, int need_storage)
538 {
539  VulkanDevicePriv *priv = dev_ctx->hwctx;
540  AVVulkanDeviceContext *hwctx = &priv->p;
541  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
542 
543  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
544  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
545  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
546 
547  for (int i = 0; i < nb_vk_formats_list; i++) {
548  if (vk_formats_list[i].pixfmt == p) {
549  VkFormatProperties3 fprops = {
550  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
551  };
552  VkFormatProperties2 prop = {
553  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
554  .pNext = &fprops,
555  };
556  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
557  int basics_primary = 0, basics_secondary = 0;
558  int storage_primary = 0, storage_secondary = 0;
559 
560  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
561  vk_formats_list[i].vkf,
562  &prop);
563 
564  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
565  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
566  basics_primary = (feats_primary & basic_flags) == basic_flags;
567  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
568 
569  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
570  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
571  vk_formats_list[i].fallback[0],
572  &prop);
573  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
574  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
575  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
576  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
577  } else {
578  basics_secondary = basics_primary;
579  storage_secondary = storage_primary;
580  }
581 
582  if (basics_primary &&
583  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
584  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
585  if (fmts) {
586  if (vk_formats_list[i].nb_images > 1) {
587  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
588  fmts[j] = vk_formats_list[i].fallback[j];
589  } else {
590  fmts[0] = vk_formats_list[i].vkf;
591  }
592  }
593  if (nb_images)
594  *nb_images = 1;
595  if (aspect)
596  *aspect = vk_formats_list[i].aspect;
597  if (supported_usage)
598  *supported_usage = ff_vk_map_feats_to_usage(feats_primary) |
599  ((need_storage && (storage_primary | storage_secondary)) ?
600  VK_IMAGE_USAGE_STORAGE_BIT : 0);
601  return 0;
602  } else if (basics_secondary &&
603  (!need_storage || (need_storage && storage_secondary))) {
604  if (fmts) {
605  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
606  fmts[j] = vk_formats_list[i].fallback[j];
607  }
608  if (nb_images)
609  *nb_images = vk_formats_list[i].nb_images_fallback;
610  if (aspect)
611  *aspect = vk_formats_list[i].aspect;
612  if (supported_usage)
613  *supported_usage = ff_vk_map_feats_to_usage(feats_secondary);
614  return 0;
615  } else {
616  return AVERROR(ENOTSUP);
617  }
618  }
619  }
620 
621  return AVERROR(EINVAL);
622 }
623 
624 #if CONFIG_VULKAN_STATIC
625 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
626  const char *pName);
627 #endif
628 
629 static int load_libvulkan(AVHWDeviceContext *ctx)
630 {
631  VulkanDevicePriv *p = ctx->hwctx;
632  AVVulkanDeviceContext *hwctx = &p->p;
633 
634 #if CONFIG_VULKAN_STATIC
635  hwctx->get_proc_addr = vkGetInstanceProcAddr;
636 #else
637  static const char *lib_names[] = {
638 #if defined(_WIN32)
639  "vulkan-1.dll",
640 #elif defined(__APPLE__)
641  "libvulkan.dylib",
642  "libvulkan.1.dylib",
643  "libMoltenVK.dylib",
644 #else
645  "libvulkan.so.1",
646  "libvulkan.so",
647 #endif
648  };
649 
650  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
651  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
652  if (p->libvulkan)
653  break;
654  }
655 
656  if (!p->libvulkan) {
657  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
658  return AVERROR_UNKNOWN;
659  }
660 
661  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
662 #endif /* CONFIG_VULKAN_STATIC */
663 
664  return 0;
665 }
666 
667 typedef struct VulkanOptExtension {
668  const char *name;
669  FFVulkanExtensions flag;
670 } VulkanOptExtension;
671 
672 static const VulkanOptExtension optional_instance_exts[] = {
673 #ifdef __APPLE__
674  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
675 #endif
676  { 0 },
677 };
678 
679 static const VulkanOptExtension optional_device_exts[] = {
680  /* Misc or required by other extensions */
681  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
682  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
683  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
684  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
685  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
686  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
687  { VK_KHR_SHADER_SUBGROUP_ROTATE_EXTENSION_NAME, FF_VK_EXT_SUBGROUP_ROTATE },
688  { VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME, FF_VK_EXT_HOST_IMAGE_COPY },
689  { VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, FF_VK_EXT_EXPLICIT_MEM_LAYOUT },
690 #ifdef VK_KHR_shader_relaxed_extended_instruction
691  { VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME, FF_VK_EXT_RELAXED_EXTENDED_INSTR },
692 #endif
693 #ifdef VK_EXT_shader_long_vector
694  { VK_EXT_SHADER_LONG_VECTOR_EXTENSION_NAME, FF_VK_EXT_LONG_VECTOR },
695 #endif
696 #ifdef VK_EXT_shader_replicated_composites
697  { VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME, FF_VK_EXT_REPLICATED_COMPOSITES },
698 #endif
699 #ifdef VK_EXT_zero_initialize_device_memory
700  { VK_EXT_ZERO_INITIALIZE_DEVICE_MEMORY_EXTENSION_NAME, FF_VK_EXT_ZERO_INITIALIZE },
701 #endif
702 #ifdef VK_KHR_shader_expect_assume
703  { VK_KHR_SHADER_EXPECT_ASSUME_EXTENSION_NAME, FF_VK_EXT_EXPECT_ASSUME },
704 #endif
705  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
706 #ifdef VK_KHR_video_maintenance2
707  { VK_KHR_VIDEO_MAINTENANCE_2_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_2 },
708 #endif
709 
710  /* Imports/exports */
711  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
712  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
713  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
714  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
715  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
716 #ifdef _WIN32
717  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
718  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
719 #endif
720 
721  /* Video encoding/decoding */
722  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
723  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
724  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
725  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
726  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
727  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
728  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
729 #ifdef VK_KHR_video_decode_vp9
730  { VK_KHR_VIDEO_DECODE_VP9_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_VP9 },
731 #endif
732 #ifdef VK_KHR_video_encode_av1
733  { VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_AV1 },
734 #endif
735  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
736 };
737 
738 const char **av_vk_get_optional_instance_extensions(int *count)
739 {
740  const char **exts = av_malloc_array(sizeof(*exts),
741  FF_ARRAY_ELEMS(optional_instance_exts) - 1);
742  if (!exts)
743  return NULL;
744 
745  for (int i = 0; i < FF_ARRAY_ELEMS(optional_instance_exts) - 1; i++)
746  exts[i] = optional_instance_exts[i].name;
747 
748  *count = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
749  return exts;
750 }
751 
752 const char **av_vk_get_optional_device_extensions(int *count)
753 {
754  const char **exts = av_malloc_array(sizeof(*exts),
755  FF_ARRAY_ELEMS(optional_device_exts));
756  if (!exts)
757  return NULL;
758 
759  for (int i = 0; i < FF_ARRAY_ELEMS(optional_device_exts); i++)
760  exts[i] = optional_device_exts[i].name;
761 
762  *count = FF_ARRAY_ELEMS(optional_device_exts);
763  return exts;
764 }
765 
766 static VKAPI_ATTR
767 VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
768  VkDebugUtilsMessageTypeFlagsEXT messageType,
769  const VkDebugUtilsMessengerCallbackDataEXT *data,
770  void *priv)
771 {
772  int l;
773  AVHWDeviceContext *ctx = priv;
774 
775  /* Ignore false positives */
776  switch (data->messageIdNumber) {
777  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
778  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
779  return VK_FALSE;
780  default:
781  break;
782  }
783 
784  switch (severity) {
785  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
786  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
787  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
788  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
789  default: l = AV_LOG_DEBUG; break;
790  }
791 
792  av_log(ctx, l, "%s\n", data->pMessage);
793  for (int i = 0; i < data->cmdBufLabelCount; i++)
794  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
795 
796  return VK_FALSE;
797 }
798 
799 #define ADD_VAL_TO_LIST(list, count, val) \
800  do { \
801  list = av_realloc_array(list, ++count, sizeof(*list)); \
802  if (!list) { \
803  err = AVERROR(ENOMEM); \
804  goto fail; \
805  } \
806  list[count - 1] = av_strdup(val); \
807  if (!list[count - 1]) { \
808  err = AVERROR(ENOMEM); \
809  goto fail; \
810  } \
811  } while(0)
812 
813 #define RELEASE_PROPS(props, count) \
814  if (props) { \
815  for (int i = 0; i < count; i++) \
816  av_free((void *)((props)[i])); \
817  av_free((void *)props); \
818  }
819 
820 static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
821 {
822  VulkanDevicePriv *p = ctx->hwctx;
823  VkDeviceSize max_vram = 0, max_visible_vram = 0;
824 
825  /* Get device memory properties */
826  av_assert0(p->mprops.memoryTypeCount);
827  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
828  const VkMemoryType type = p->mprops.memoryTypes[i];
829  const VkMemoryHeap heap = p->mprops.memoryHeaps[type.heapIndex];
830  if (!(type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
831  continue;
832  max_vram = FFMAX(max_vram, heap.size);
833  if (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
834  max_visible_vram = FFMAX(max_visible_vram, heap.size);
835  }
836 
837  return max_vram - max_visible_vram < 1024; /* 1 kB tolerance */
838 }
839 
840 enum FFVulkanDebugMode {
841  FF_VULKAN_DEBUG_NONE = 0,
842  /* Standard GPU-assisted validation */
843  FF_VULKAN_DEBUG_VALIDATE = 1,
844  /* Passes printfs in shaders to the debug callback */
845  FF_VULKAN_DEBUG_PRINTF = 2,
846  /* Enables extra printouts */
847  FF_VULKAN_DEBUG_PRACTICES = 3,
848 
849  FF_VULKAN_DEBUG_NB,
850 };
851 
852 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
853  const char * const **dst, uint32_t *num,
854  enum FFVulkanDebugMode debug_mode)
855 {
856  const char *tstr;
857  const char **extension_names = NULL;
858  VulkanDevicePriv *p = ctx->hwctx;
859  AVVulkanDeviceContext *hwctx = &p->p;
860  FFVulkanFunctions *vk = &p->vkctx.vkfn;
861  int err = 0, found, extensions_found = 0;
862 
863  const char *mod;
864  int optional_exts_num;
865  uint32_t sup_ext_count;
866  char *user_exts_str = NULL;
867  AVDictionaryEntry *user_exts;
868  VkExtensionProperties *sup_ext;
869  const VulkanOptExtension *optional_exts;
870 
871  if (!dev) {
872  mod = "instance";
873  optional_exts = optional_instance_exts;
874  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
875  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
876  if (user_exts) {
877  user_exts_str = av_strdup(user_exts->value);
878  if (!user_exts_str) {
879  err = AVERROR(ENOMEM);
880  goto fail;
881  }
882  }
883  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
884  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
885  if (!sup_ext)
886  return AVERROR(ENOMEM);
887  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
888  } else {
889  mod = "device";
890  optional_exts = optional_device_exts;
891  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
892  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
893  if (user_exts) {
894  user_exts_str = av_strdup(user_exts->value);
895  if (!user_exts_str) {
896  err = AVERROR(ENOMEM);
897  goto fail;
898  }
899  }
900  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
901  &sup_ext_count, NULL);
902  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
903  if (!sup_ext)
904  return AVERROR(ENOMEM);
905  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
906  &sup_ext_count, sup_ext);
907  }
908 
909  for (int i = 0; i < optional_exts_num; i++) {
910  tstr = optional_exts[i].name;
911  found = 0;
912 
913  /* Check if the device has ReBAR for host image copies */
914  if (!strcmp(tstr, VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME) &&
915  !vulkan_device_has_rebar(ctx))
916  continue;
917 
918  if (dev &&
919  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
920  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
921  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
922  (!strcmp(tstr, VK_EXT_SHADER_OBJECT_EXTENSION_NAME))) {
923  continue;
924  }
925 
926  for (int j = 0; j < sup_ext_count; j++) {
927  if (!strcmp(tstr, sup_ext[j].extensionName)) {
928  found = 1;
929  break;
930  }
931  }
932  if (!found)
933  continue;
934 
935  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
936  p->vkctx.extensions |= optional_exts[i].flag;
937  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
938  }
939 
940  if (!dev &&
941  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
942  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
943  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
944  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
945  found = 0;
946  for (int j = 0; j < sup_ext_count; j++) {
947  if (!strcmp(tstr, sup_ext[j].extensionName)) {
948  found = 1;
949  break;
950  }
951  }
952  if (found) {
953  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
954  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
955  } else {
956  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
957  tstr);
958  err = AVERROR(EINVAL);
959  goto fail;
960  }
961  }
962 
963 #ifdef VK_KHR_shader_relaxed_extended_instruction
964  if ((debug_mode == FF_VULKAN_DEBUG_PRINTF) && dev) {
965  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
966  found = 0;
967  for (int j = 0; j < sup_ext_count; j++) {
968  if (!strcmp(tstr, sup_ext[j].extensionName)) {
969  found = 1;
970  break;
971  }
972  }
973  if (!found) {
974  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
975  tstr);
976  err = AVERROR(EINVAL);
977  goto fail;
978  }
979  }
980 #endif
981 
982  if (user_exts_str) {
983  char *save, *token = av_strtok(user_exts_str, "+", &save);
984  while (token) {
985  found = 0;
986  for (int j = 0; j < sup_ext_count; j++) {
987  if (!strcmp(token, sup_ext[j].extensionName)) {
988  found = 1;
989  break;
990  }
991  }
992  if (found) {
993  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
994  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
995  } else {
996  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
997  mod, token);
998  }
999  token = av_strtok(NULL, "+", &save);
1000  }
1001  }
1002 
1003  *dst = extension_names;
1004  *num = extensions_found;
1005 
1006  av_free(user_exts_str);
1007  av_free(sup_ext);
1008  return 0;
1009 
1010 fail:
1011  RELEASE_PROPS(extension_names, extensions_found);
1012  av_free(user_exts_str);
1013  av_free(sup_ext);
1014  return err;
1015 }
1016 
1017 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
1018  const char * const **dst, uint32_t *num,
1019  enum FFVulkanDebugMode *debug_mode)
1020 {
1021  int err = 0;
1022  VulkanDevicePriv *priv = ctx->hwctx;
1023  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
1024 
1025  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
1026  int layer_standard_validation_found = 0;
1027 
1028  uint32_t sup_layer_count;
1029  VkLayerProperties *sup_layers;
1030 
1031  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
1032  char *user_layers_str = NULL;
1033  char *save, *token;
1034 
1035  const char **enabled_layers = NULL;
1036  uint32_t enabled_layers_count = 0;
1037 
1038  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
1039  enum FFVulkanDebugMode mode;
1040 
1041  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
1042 
1043  /* Get a list of all layers */
1044  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
1045  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
1046  if (!sup_layers)
1047  return AVERROR(ENOMEM);
1048  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
1049 
1050  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
1051  for (int i = 0; i < sup_layer_count; i++)
1052  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
1053 
1054  /* If no user layers or debug layers are given, return */
1055  if (!debug_opt && !user_layers)
1056  goto end;
1057 
1058  /* Check for any properly supported validation layer */
1059  if (debug_opt) {
1060  if (!strcmp(debug_opt->value, "printf")) {
1061  mode = FF_VULKAN_DEBUG_PRINTF;
1062  } else if (!strcmp(debug_opt->value, "validate")) {
1063  mode = FF_VULKAN_DEBUG_VALIDATE;
1064  } else if (!strcmp(debug_opt->value, "practices")) {
1065  mode = FF_VULKAN_DEBUG_PRACTICES;
1066  } else {
1067  char *end_ptr = NULL;
1068  int idx = strtol(debug_opt->value, &end_ptr, 10);
1069  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
1070  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
1071  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
1072  debug_opt->value);
1073  err = AVERROR(EINVAL);
1074  goto end;
1075  }
1076  mode = idx;
1077  }
1078  }
1079 
1080  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
1081  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
1082  (mode == FF_VULKAN_DEBUG_PRINTF) ||
1083  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
1084  for (int i = 0; i < sup_layer_count; i++) {
1085  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
1086  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
1087  layer_standard_validation);
1088  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
1089  *debug_mode = mode;
1090  layer_standard_validation_found = 1;
1091  break;
1092  }
1093  }
1094  if (!layer_standard_validation_found) {
1095  av_log(ctx, AV_LOG_ERROR,
1096  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
1097  err = AVERROR(ENOTSUP);
1098  goto end;
1099  }
1100  }
1101 
1102  /* Process any custom layers enabled */
1103  if (user_layers) {
1104  int found;
1105 
1106  user_layers_str = av_strdup(user_layers->value);
1107  if (!user_layers_str) {
1108  err = AVERROR(ENOMEM);
1109  goto fail;
1110  }
1111 
1112  token = av_strtok(user_layers_str, "+", &save);
1113  while (token) {
1114  found = 0;
1115 
1116  /* If debug=1/2 was specified as an option, skip this layer */
1117  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
1118  token = av_strtok(NULL, "+", &save);
1119  break;
1120  }
1121 
1122  /* Try to find the layer in the list of supported layers */
1123  for (int j = 0; j < sup_layer_count; j++) {
1124  if (!strcmp(token, sup_layers[j].layerName)) {
1125  found = 1;
1126  break;
1127  }
1128  }
1129 
1130  if (found) {
1131  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
1132  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
1133 
1134  /* If debug was not set as an option, force it */
1135  if (!strcmp(layer_standard_validation, token))
1136  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
1137  } else {
1138  av_log(ctx, AV_LOG_ERROR,
1139  "Layer \"%s\" not supported\n", token);
1140  err = AVERROR(EINVAL);
1141  goto end;
1142  }
1143 
1144  token = av_strtok(NULL, "+", &save);
1145  }
1146  }
1147 
1148 fail:
1149 end:
1150  av_free(sup_layers);
1151  av_free(user_layers_str);
1152 
1153  if (err < 0) {
1154  RELEASE_PROPS(enabled_layers, enabled_layers_count);
1155  } else {
1156  *dst = enabled_layers;
1157  *num = enabled_layers_count;
1158  }
1159 
1160  return err;
1161 }
1162 
1163 /* Creates a VkInstance */
1164 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
1165  enum FFVulkanDebugMode *debug_mode)
1166 {
1167  int err = 0;
1168  VkResult ret;
1169  VulkanDevicePriv *p = ctx->hwctx;
1170  AVVulkanDeviceContext *hwctx = &p->p;
1171  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1172  VkApplicationInfo application_info = {
1173  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1174  .pApplicationName = "ffmpeg",
1175  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1176  LIBAVUTIL_VERSION_MINOR,
1177  LIBAVUTIL_VERSION_MICRO),
1178  .pEngineName = "libavutil",
1179  .apiVersion = VK_API_VERSION_1_3,
1180  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1181  LIBAVUTIL_VERSION_MINOR,
1182  LIBAVUTIL_VERSION_MICRO),
1183  };
1184  VkValidationFeaturesEXT validation_features = {
1185  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1186  };
1187  VkInstanceCreateInfo inst_props = {
1188  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1189  .pApplicationInfo = &application_info,
1190  };
1191 
1192  if (!hwctx->get_proc_addr) {
1193  err = load_libvulkan(ctx);
1194  if (err < 0)
1195  return err;
1196  }
1197 
1198  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1199  if (err < 0) {
1200  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1201  return err;
1202  }
1203 
1204  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1205  &inst_props.enabledLayerCount, debug_mode);
1206  if (err)
1207  goto fail;
1208 
1209  /* Check for present/missing extensions */
1210  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1211  &inst_props.enabledExtensionCount, *debug_mode);
1212  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1213  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1214  if (err < 0)
1215  goto fail;
1216 
1217  /* Enable debug features if needed */
1218  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1219  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1220  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1221  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1222  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1223  };
1224  validation_features.pEnabledValidationFeatures = feat_list_validate;
1225  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1226  inst_props.pNext = &validation_features;
1227  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1228  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1229  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1230  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1231  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1232  };
1233  validation_features.pEnabledValidationFeatures = feat_list_debug;
1234  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1235  inst_props.pNext = &validation_features;
1236  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1237  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1238  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1239  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1240  };
1241  validation_features.pEnabledValidationFeatures = feat_list_practices;
1242  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1243  inst_props.pNext = &validation_features;
1244  }
1245 
1246 #ifdef __APPLE__
1247  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1248  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1249  inst_props.ppEnabledExtensionNames[i])) {
1250  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1251  break;
1252  }
1253  }
1254 #endif
1255 
1256  /* Try to create the instance */
1257  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1258 
1259  /* Check for errors */
1260  if (ret != VK_SUCCESS) {
1261  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1262  ff_vk_ret2str(ret));
1263  err = AVERROR_EXTERNAL;
1264  goto fail;
1265  }
1266 
1267  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1268  if (err < 0) {
1269  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1270  goto fail;
1271  }
1272 
1273  /* Setup debugging callback if needed */
1274  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1275  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1276  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1277  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1278  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1279  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1280  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1281  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1282  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1283  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1284  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1285  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1286  .pfnUserCallback = vk_dbg_callback,
1287  .pUserData = ctx,
1288  };
1289 
1290  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1291  hwctx->alloc, &p->debug_ctx);
1292  }
1293 
1294  err = 0;
1295 
1296 fail:
1297  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1298  return err;
1299 }
1300 
1301 typedef struct VulkanDeviceSelection {
1302  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1303  int has_uuid;
1304  uint32_t drm_major; /* Will use this second unless !has_drm */
1305  uint32_t drm_minor; /* Will use this second unless !has_drm */
1306  uint32_t has_drm; /* has drm node info */
1307  const char *name; /* Will use this third unless NULL */
1308  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1309  uint32_t vendor_id; /* Last resort to find something deterministic */
1310  int index; /* Finally fall back to index */
1311 } VulkanDeviceSelection;
1312 
1313 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1314 {
1315  switch (type) {
1316  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1317  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1318  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1319  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1320  default: return "unknown";
1321  }
1322 }
1323 
1324 /* Finds a device */
1325 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1326 {
1327  int err = 0, choice = -1;
1328  uint32_t num;
1329  VkResult ret;
1330  VulkanDevicePriv *p = ctx->hwctx;
1331  AVVulkanDeviceContext *hwctx = &p->p;
1332  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1333  VkPhysicalDevice *devices = NULL;
1334  VkPhysicalDeviceIDProperties *idp = NULL;
1335  VkPhysicalDeviceProperties2 *prop = NULL;
1336  VkPhysicalDeviceDriverProperties *driver_prop = NULL;
1337  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1338 
1339  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1340  if (ret != VK_SUCCESS || !num) {
1341  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1342  return AVERROR(ENODEV);
1343  }
1344 
1345  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1346  if (!devices)
1347  return AVERROR(ENOMEM);
1348 
1349  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1350  if (ret != VK_SUCCESS) {
1351  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1352  ff_vk_ret2str(ret));
1353  err = AVERROR(ENODEV);
1354  goto end;
1355  }
1356 
1357  prop = av_calloc(num, sizeof(*prop));
1358  if (!prop) {
1359  err = AVERROR(ENOMEM);
1360  goto end;
1361  }
1362 
1363  idp = av_calloc(num, sizeof(*idp));
1364  if (!idp) {
1365  err = AVERROR(ENOMEM);
1366  goto end;
1367  }
1368 
1369  driver_prop = av_calloc(num, sizeof(*driver_prop));
1370  if (!driver_prop) {
1371  err = AVERROR(ENOMEM);
1372  goto end;
1373  }
1374 
1375  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1376  drm_prop = av_calloc(num, sizeof(*drm_prop));
1377  if (!drm_prop) {
1378  err = AVERROR(ENOMEM);
1379  goto end;
1380  }
1381  }
1382 
1383  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1384  for (int i = 0; i < num; i++) {
1385  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1386  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1387  driver_prop[i].pNext = &drm_prop[i];
1388  }
1389  driver_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1390  idp[i].pNext = &driver_prop[i];
1391  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1392  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1393  prop[i].pNext = &idp[i];
1394 
1395  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1396  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1397  prop[i].properties.deviceName,
1398  vk_dev_type(prop[i].properties.deviceType),
1399  prop[i].properties.deviceID);
1400  }
1401 
1402  if (select->has_uuid) {
1403  for (int i = 0; i < num; i++) {
1404  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1405  choice = i;
1406  goto end;
1407  }
1408  }
1409  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1410  err = AVERROR(ENODEV);
1411  goto end;
1412  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1413  for (int i = 0; i < num; i++) {
1414  if ((select->drm_major == drm_prop[i].primaryMajor &&
1415  select->drm_minor == drm_prop[i].primaryMinor) ||
1416  (select->drm_major == drm_prop[i].renderMajor &&
1417  select->drm_minor == drm_prop[i].renderMinor)) {
1418  choice = i;
1419  goto end;
1420  }
1421  }
1422  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1423  select->drm_major, select->drm_minor);
1424  err = AVERROR(ENODEV);
1425  goto end;
1426  } else if (select->name) {
1427  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1428  for (int i = 0; i < num; i++) {
1429  if (strstr(prop[i].properties.deviceName, select->name)) {
1430  choice = i;
1431  goto end;
1432  }
1433  }
1434  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1435  select->name);
1436  err = AVERROR(ENODEV);
1437  goto end;
1438  } else if (select->pci_device) {
1439  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1440  for (int i = 0; i < num; i++) {
1441  if (select->pci_device == prop[i].properties.deviceID) {
1442  choice = i;
1443  goto end;
1444  }
1445  }
1446  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1447  select->pci_device);
1448  err = AVERROR(EINVAL);
1449  goto end;
1450  } else if (select->vendor_id) {
1451  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1452  for (int i = 0; i < num; i++) {
1453  if (select->vendor_id == prop[i].properties.vendorID) {
1454  choice = i;
1455  goto end;
1456  }
1457  }
1458  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1459  select->vendor_id);
1460  err = AVERROR(ENODEV);
1461  goto end;
1462  } else {
1463  if (select->index < num) {
1464  choice = select->index;
1465  goto end;
1466  }
1467  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1468  select->index);
1469  err = AVERROR(ENODEV);
1470  goto end;
1471  }
1472 
1473 end:
1474  if (choice > -1) {
1475  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1476  choice, prop[choice].properties.deviceName,
1477  vk_dev_type(prop[choice].properties.deviceType),
1478  prop[choice].properties.deviceID);
1479  hwctx->phys_dev = devices[choice];
1480  p->props = prop[choice];
1481  p->props.pNext = NULL;
1482  p->dprops = driver_prop[choice];
1483  p->dprops.pNext = NULL;
1484  }
1485 
1486  av_free(devices);
1487  av_free(prop);
1488  av_free(idp);
1489  av_free(drm_prop);
1490  av_free(driver_prop);
1491 
1492  return err;
1493 }
1494 
1495 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1496 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1497  VkQueueFlagBits flags)
1498 {
1499  int index = -1;
1500  uint32_t min_score = UINT32_MAX;
1501 
1502  for (int i = 0; i < num_qf; i++) {
1503  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1504 
1505  /* Per the spec, reporting transfer caps is optional for these 2 types */
1506  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1507  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1508  qflags |= VK_QUEUE_TRANSFER_BIT;
1509 
1510  if (qflags & flags) {
1511  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1512  if (score < min_score) {
1513  index = i;
1514  min_score = score;
1515  }
1516  }
1517  }
1518 
1519  if (index > -1)
1520  qf[index].queueFamilyProperties.timestampValidBits++;
1521 
1522  return index;
1523 }
1524 
1525 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1526  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1527  VkVideoCodecOperationFlagsKHR flags)
1528 {
1529  int index = -1;
1530  uint32_t min_score = UINT32_MAX;
1531 
1532  for (int i = 0; i < num_qf; i++) {
1533  const VkQueueFlags qflags = qf[i].queueFamilyProperties.queueFlags;
1534  const VkVideoCodecOperationFlagsKHR vflags = qf_vid[i].videoCodecOperations;
1535 
1536  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1537  continue;
1538 
1539  if (vflags & flags) {
1540  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1541  if (score < min_score) {
1542  index = i;
1543  min_score = score;
1544  }
1545  }
1546  }
1547 
1548  if (index > -1)
1549  qf[index].queueFamilyProperties.timestampValidBits++;
1550 
1551  return index;
1552 }
1553 
1554 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1555 {
1556  uint32_t num;
1557  VulkanDevicePriv *p = ctx->hwctx;
1558  AVVulkanDeviceContext *hwctx = &p->p;
1559  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1560 
1561  VkQueueFamilyProperties2 *qf = NULL;
1562  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1563 
1564  /* First get the number of queue families */
1565  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1566  if (!num) {
1567  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1568  return AVERROR_EXTERNAL;
1569  }
1570 
1571  /* Then allocate memory */
1572  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1573  if (!qf)
1574  return AVERROR(ENOMEM);
1575 
1576  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1577  if (!qf_vid)
1578  return AVERROR(ENOMEM);
1579 
1580  for (uint32_t i = 0; i < num; i++) {
1581  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1582  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1583  };
1584  qf[i] = (VkQueueFamilyProperties2) {
1585  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1586  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1587  };
1588  }
1589 
1590  /* Finally retrieve the queue families */
1591  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1592 
1593  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1594  for (int i = 0; i < num; i++) {
1595  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1596  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1597  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1598  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1599  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1600  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1601  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1602  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1603  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1604  qf[i].queueFamilyProperties.queueCount);
1605 
1606  /* We use this field to keep a score of how many times we've used that
1607  * queue family in order to make better choices. */
1608  qf[i].queueFamilyProperties.timestampValidBits = 0;
1609  }
1610 
1611  hwctx->nb_qf = 0;
1612 
1613  /* Pick each queue family to use. */
1614 #define PICK_QF(type, vid_op) \
1615  do { \
1616  uint32_t i; \
1617  uint32_t idx; \
1618  \
1619  if (vid_op) \
1620  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1621  else \
1622  idx = pick_queue_family(qf, num, type); \
1623  \
1624  if (idx == -1) \
1625  continue; \
1626  \
1627  for (i = 0; i < hwctx->nb_qf; i++) { \
1628  if (hwctx->qf[i].idx == idx) { \
1629  hwctx->qf[i].flags |= type; \
1630  hwctx->qf[i].video_caps |= vid_op; \
1631  break; \
1632  } \
1633  } \
1634  if (i == hwctx->nb_qf) { \
1635  hwctx->qf[i].idx = idx; \
1636  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1637  if (p->limit_queues || \
1638  p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) { \
1639  int max = p->limit_queues; \
1640  if (type == VK_QUEUE_GRAPHICS_BIT) \
1641  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, \
1642  max ? max : 1); \
1643  else if (max) \
1644  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, max); \
1645  } \
1646  hwctx->qf[i].flags = type; \
1647  hwctx->qf[i].video_caps = vid_op; \
1648  hwctx->nb_qf++; \
1649  } \
1650  } while (0)
1651 
1652  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1653  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1654  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1655 
1656  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1657  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1658 
1659  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1660  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1661 
1662 #ifdef VK_KHR_video_decode_vp9
1663  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_VP9_BIT_KHR);
1664 #endif
1665 
1666 #ifdef VK_KHR_video_encode_av1
1667  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR);
1668 #endif
1669  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1670 
1671  av_free(qf);
1672  av_free(qf_vid);
1673 
1674 #undef PICK_QF
1675 
1676  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1677  sizeof(VkDeviceQueueCreateInfo));
1678  if (!cd->pQueueCreateInfos)
1679  return AVERROR(ENOMEM);
1680 
1681  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1682  int dup = 0;
1683  float *weights = NULL;
1684  VkDeviceQueueCreateInfo *pc;
1685  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1686  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1687  dup = 1;
1688  break;
1689  }
1690  }
1691  if (dup)
1692  continue;
1693 
1694  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1695  if (!weights) {
1696  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1697  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1698  av_free((void *)cd->pQueueCreateInfos);
1699  return AVERROR(ENOMEM);
1700  }
1701 
1702  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1703  weights[j] = 1.0;
1704 
1705  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1706  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1707  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1708  .queueFamilyIndex = hwctx->qf[i].idx,
1709  .queueCount = hwctx->qf[i].num,
1710  .pQueuePriorities = weights,
1711  };
1712  }
1713 
1714 #if FF_API_VULKAN_FIXED_QUEUES
1715 FF_DISABLE_DEPRECATION_WARNINGS
1716  /* Setup deprecated fields */
1717  hwctx->queue_family_index = -1;
1718  hwctx->queue_family_comp_index = -1;
1719  hwctx->queue_family_tx_index = -1;
1720  hwctx->queue_family_encode_index = -1;
1721  hwctx->queue_family_decode_index = -1;
1722 
1723 #define SET_OLD_QF(field, nb_field, type) \
1724  do { \
1725  if (field < 0 && hwctx->qf[i].flags & type) { \
1726  field = hwctx->qf[i].idx; \
1727  nb_field = hwctx->qf[i].num; \
1728  } \
1729  } while (0)
1730 
1731  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1732  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1733  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1734  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1735  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1736  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1737  }
1738 
1739 #undef SET_OLD_QF
1740 FF_ENABLE_DEPRECATION_WARNINGS
1741 #endif
1742 
1743  return 0;
1744 }
1745 
1746 /* Only resources created by vulkan_device_create should be released here,
1747  * resources created by vulkan_device_init should be released by
1748  * vulkan_device_uninit, to make sure we don't free user provided resources,
1749  * and there is no leak.
1750  */
1751 static void vulkan_device_free(AVHWDeviceContext *ctx)
1752 {
1753  VulkanDevicePriv *p = ctx->hwctx;
1754  AVVulkanDeviceContext *hwctx = &p->p;
1755  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1756 
1757  if (hwctx->act_dev)
1758  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1759 
1760  if (p->debug_ctx)
1761  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1762  hwctx->alloc);
1763 
1764  if (hwctx->inst)
1765  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1766 
1767  if (p->libvulkan)
1768  dlclose(p->libvulkan);
1769 
1770  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1771  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1772 }
1773 
1774 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1775 {
1776  VulkanDevicePriv *p = ctx->hwctx;
1777 
1778  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1779  pthread_mutex_destroy(p->qf_mutex[i]);
1780  av_freep(&p->qf_mutex[i]);
1781  }
1782  av_freep(&p->qf_mutex);
1783 
1784  ff_vk_uninit(&p->vkctx);
1785 }
1786 
1787 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1788  VulkanDeviceSelection *dev_select,
1789  int disable_multiplane,
1790  AVDictionary *opts, int flags)
1791 {
1792  int err = 0;
1793  VkResult ret;
1794  AVDictionaryEntry *opt_d;
1795  VulkanDevicePriv *p = ctx->hwctx;
1796  AVVulkanDeviceContext *hwctx = &p->p;
1797  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1798  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1799  VulkanDeviceFeatures supported_feats = { 0 };
1800  VkDeviceCreateInfo dev_info = {
1801  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1802  };
1803 
1804  /* Create an instance if not given one */
1805  if ((err = create_instance(ctx, opts, &debug_mode)))
1806  goto end;
1807 
1808  /* Find a physical device (if not given one) */
1809  if ((err = find_device(ctx, dev_select)))
1810  goto end;
1811 
1812  /* Get supported memory types */
1813  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1814 
1815  /* Find and enable extensions for the physical device */
1816  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1817  &dev_info.enabledExtensionCount, debug_mode))) {
1818  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1819  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1820  av_free((void *)dev_info.pQueueCreateInfos);
1821  goto end;
1822  }
1823 
1824  /* Get all supported features for the physical device */
1825  device_features_init(ctx, &supported_feats);
1826  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1827 
1828  /* Copy all needed features from those supported and activate them */
1829  device_features_init(ctx, &p->feats);
1830  device_features_copy_needed(&p->feats, &supported_feats);
1831  dev_info.pNext = p->feats.device.pNext;
1832  dev_info.pEnabledFeatures = &p->feats.device.features;
1833 
1834  /* Limit queues to a given number if needed */
1835  opt_d = av_dict_get(opts, "limit_queues", NULL, 0);
1836  if (opt_d)
1837  p->limit_queues = strtol(opt_d->value, NULL, 10);
1838 
1839  /* Setup enabled queue families */
1840  if ((err = setup_queue_families(ctx, &dev_info)))
1841  goto end;
1842 
1843  /* Finally create the device */
1844  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1845  &hwctx->act_dev);
1846 
1847  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1848  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1849  av_free((void *)dev_info.pQueueCreateInfos);
1850 
1851  if (ret != VK_SUCCESS) {
1852  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1853  ff_vk_ret2str(ret));
1854  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1855  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1856  av_free((void *)dev_info.ppEnabledExtensionNames);
1857  err = AVERROR_EXTERNAL;
1858  goto end;
1859  }
1860 
1861  /* Tiled images setting, use them by default */
1862  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1863  if (opt_d)
1864  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1865 
1866  /* The disable_multiplane argument takes precedent over the option */
1867  p->disable_multiplane = disable_multiplane;
1868  if (!p->disable_multiplane) {
1869  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1870  if (opt_d)
1871  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1872  }
1873 
1874  /* Disable host pointer imports (by default on nvidia) */
1875  p->avoid_host_import = p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
1876  opt_d = av_dict_get(opts, "avoid_host_import", NULL, 0);
1877  if (opt_d)
1878  p->avoid_host_import = strtol(opt_d->value, NULL, 10);
1879 
1880  /* Set the public device feature struct and its pNext chain */
1881  hwctx->device_features = p->feats.device;
1882 
1883  /* Set the list of all active extensions */
1884  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1885  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1886 
1887  /* The extension lists need to be freed */
1888  ctx->free = vulkan_device_free;
1889 
1890 end:
1891  return err;
1892 }
1893 
1894 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1895 {
1896  VulkanDevicePriv *p = ctx->hwctx;
1897  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1898 }
1899 
1900 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1901 {
1902  VulkanDevicePriv *p = ctx->hwctx;
1903  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1904 }
1905 
1906 static int vulkan_device_init(AVHWDeviceContext *ctx)
1907 {
1908  int err = 0;
1909  uint32_t qf_num;
1910  VulkanDevicePriv *p = ctx->hwctx;
1911  AVVulkanDeviceContext *hwctx = &p->p;
1912  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1913  VkQueueFamilyProperties2 *qf;
1914  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1915  VkPhysicalDeviceExternalSemaphoreInfo ext_sem_props_info;
1916  int graph_index, comp_index, tx_index, enc_index, dec_index;
1917 
1918  /* Set device extension flags */
1919  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1920  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1921  if (!strcmp(hwctx->enabled_dev_extensions[i],
1922  optional_device_exts[j].name)) {
1923  p->vkctx.extensions |= optional_device_exts[j].flag;
1924  break;
1925  }
1926  }
1927  }
1928 
1929  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1930  if (err < 0) {
1931  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1932  return err;
1933  }
1934 
1935  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1936  p->props.pNext = &p->hprops;
1937  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1938  p->hprops.pNext = &p->dprops;
1939  p->dprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1940 
1941  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1942  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1943  p->props.properties.deviceName);
1944  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1945  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1946  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1947  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %zu\n",
1948  p->props.properties.limits.minMemoryMapAlignment);
1949  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1950  p->props.properties.limits.nonCoherentAtomSize);
1951  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1952  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1953  p->hprops.minImportedHostPointerAlignment);
1954 
1955  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1956  if (!qf_num) {
1957  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1958  return AVERROR_EXTERNAL;
1959  }
1960 
1961  ext_sem_props_info = (VkPhysicalDeviceExternalSemaphoreInfo) {
1962  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO,
1963  };
1964 
1965  /* Opaque FD semaphore properties */
1966  ext_sem_props_info.handleType =
1967 #ifdef _WIN32
1968  IsWindows8OrGreater()
1969  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
1970  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT;
1971 #else
1972  VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
1973 #endif
1974  p->ext_sem_props_opaque.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
1975  vk->GetPhysicalDeviceExternalSemaphoreProperties(hwctx->phys_dev,
1976  &ext_sem_props_info,
1977  &p->ext_sem_props_opaque);
1978 
1979  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1980  if (!qf)
1981  return AVERROR(ENOMEM);
1982 
1983  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1984  if (!qf_vid) {
1985  av_free(qf);
1986  return AVERROR(ENOMEM);
1987  }
1988 
1989  for (uint32_t i = 0; i < qf_num; i++) {
1990  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1991  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1992  };
1993  qf[i] = (VkQueueFamilyProperties2) {
1994  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1995  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1996  };
1997  }
1998 
1999  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
2000 
2001  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
2002  if (!p->qf_mutex) {
2003  err = AVERROR(ENOMEM);
2004  goto end;
2005  }
2006  p->nb_tot_qfs = qf_num;
2007 
2008  for (uint32_t i = 0; i < qf_num; i++) {
2009  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
2010  sizeof(**p->qf_mutex));
2011  if (!p->qf_mutex[i]) {
2012  err = AVERROR(ENOMEM);
2013  goto end;
2014  }
2015  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
2016  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
2017  if (err != 0) {
2018  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
2019  av_err2str(err));
2020  err = AVERROR(err);
2021  goto end;
2022  }
2023  }
2024  }
2025 
2026 #if FF_API_VULKAN_FIXED_QUEUES
2027 FF_DISABLE_DEPRECATION_WARNINGS
2028  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
2029  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
2030  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
2031  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
2032  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
2033 
2034 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
2035  do { \
2036  if (ctx_qf < 0 && required) { \
2037  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
2038  " in the context!\n", type); \
2039  err = AVERROR(EINVAL); \
2040  goto end; \
2041  } else if (fidx < 0 || ctx_qf < 0) { \
2042  break; \
2043  } else if (ctx_qf >= qf_num) { \
2044  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
2045  type, ctx_qf, qf_num); \
2046  err = AVERROR(EINVAL); \
2047  goto end; \
2048  } \
2049  \
2050  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
2051  " for%s%s%s%s%s\n", \
2052  ctx_qf, qc, \
2053  ctx_qf == graph_index ? " graphics" : "", \
2054  ctx_qf == comp_index ? " compute" : "", \
2055  ctx_qf == tx_index ? " transfers" : "", \
2056  ctx_qf == enc_index ? " encode" : "", \
2057  ctx_qf == dec_index ? " decode" : ""); \
2058  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
2059  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
2060  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
2061  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
2062  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
2063  } while (0)
2064 
2065  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
2066  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
2067  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
2068  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
2069  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
2070 
2071 #undef CHECK_QUEUE
2072 
2073  /* Update the new queue family fields. If non-zero already,
2074  * it means API users have set it. */
2075  if (!hwctx->nb_qf) {
2076 #define ADD_QUEUE(ctx_qf, qc, flag) \
2077  do { \
2078  if (ctx_qf != -1) { \
2079  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
2080  .idx = ctx_qf, \
2081  .num = qc, \
2082  .flags = flag, \
2083  }; \
2084  } \
2085  } while (0)
2086 
2087  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
2088  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
2089  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
2090  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
2091  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
2092 #undef ADD_QUEUE
2093  }
2094 FF_ENABLE_DEPRECATION_WARNINGS
2095 #endif
2096 
2097  for (int i = 0; i < hwctx->nb_qf; i++) {
2098  if (!hwctx->qf[i].video_caps &&
2099  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
2100  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
2101  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
2102  }
2103  }
2104 
2105  /* Setup array for pQueueFamilyIndices with used queue families */
2106  p->nb_img_qfs = 0;
2107  for (int i = 0; i < hwctx->nb_qf; i++) {
2108  int seen = 0;
2109  /* Make sure each entry is unique
2110  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
2111  for (int j = (i - 1); j >= 0; j--) {
2112  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
2113  seen = 1;
2114  break;
2115  }
2116  }
2117  if (!seen)
2118  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
2119  }
2120 
2121 #if FF_API_VULKAN_SYNC_QUEUES
2122 FF_DISABLE_DEPRECATION_WARNINGS
2123  if (!hwctx->lock_queue)
2124  hwctx->lock_queue = lock_queue;
2125  if (!hwctx->unlock_queue)
2126  hwctx->unlock_queue = unlock_queue;
2127 FF_ENABLE_DEPRECATION_WARNINGS
2128 #endif
2129 
2130  /* Re-query device capabilities, in case the device was created externally */
2131  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2132 
2133  p->vkctx.device = ctx;
2134  p->vkctx.hwctx = hwctx;
2135 
2136  ff_vk_load_props(&p->vkctx);
2137  p->compute_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_COMPUTE_BIT, 0);
2138  p->transfer_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_TRANSFER_BIT, 0);
2139 
2140  /* Re-query device capabilities, in case the device was created externally */
2141  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2142 
2143 end:
2144  av_free(qf_vid);
2145  av_free(qf);
2146  return err;
2147 }
2148 
2149 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
2150  AVDictionary *opts, int flags)
2151 {
2152  VulkanDeviceSelection dev_select = { 0 };
2153  if (device && device[0]) {
2154  char *end = NULL;
2155  dev_select.index = strtol(device, &end, 10);
2156  if (end == device) {
2157  dev_select.index = 0;
2158  dev_select.name = device;
2159  }
2160  }
2161 
2162  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2163 }
2164 
2165 static int vulkan_device_derive(AVHWDeviceContext *ctx,
2166  AVHWDeviceContext *src_ctx,
2167  AVDictionary *opts, int flags)
2168 {
2169  av_unused VulkanDeviceSelection dev_select = { 0 };
2170 
2171  /* If there's only one device on the system, then even if its not covered
2172  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
2173  * dev_select will mean it'll get picked. */
2174  switch(src_ctx->type) {
2175 #if CONFIG_VAAPI
2176  case AV_HWDEVICE_TYPE_VAAPI: {
2177  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
2178  VADisplay dpy = src_hwctx->display;
2179 #if VA_CHECK_VERSION(1, 15, 0)
2180  VAStatus vas;
2181  VADisplayAttribute attr = {
2182  .type = VADisplayPCIID,
2183  };
2184 #endif
2185  const char *vendor;
2186 
2187 #if VA_CHECK_VERSION(1, 15, 0)
2188  vas = vaGetDisplayAttributes(dpy, &attr, 1);
2189  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
2190  dev_select.pci_device = (attr.value & 0xFFFF);
2191 #endif
2192 
2193  if (!dev_select.pci_device) {
2194  vendor = vaQueryVendorString(dpy);
2195  if (!vendor) {
2196  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
2197  return AVERROR_EXTERNAL;
2198  }
2199 
2200  if (strstr(vendor, "AMD"))
2201  dev_select.vendor_id = 0x1002;
2202  }
2203 
2204  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2205  }
2206 #endif
2207 #if CONFIG_LIBDRM
2208  case AV_HWDEVICE_TYPE_DRM: {
2209  int err;
2210  struct stat drm_node_info;
2211  drmDevice *drm_dev_info;
2212  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
2213 
2214  err = fstat(src_hwctx->fd, &drm_node_info);
2215  if (err) {
2216  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
2217  av_err2str(AVERROR(errno)));
2218  return AVERROR_EXTERNAL;
2219  }
2220 
2221  dev_select.drm_major = major(drm_node_info.st_dev);
2222  dev_select.drm_minor = minor(drm_node_info.st_dev);
2223  dev_select.has_drm = 1;
2224 
2225  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
2226  if (err) {
2227  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
2228  av_err2str(AVERROR(errno)));
2229  return AVERROR_EXTERNAL;
2230  }
2231 
2232  if (drm_dev_info->bustype == DRM_BUS_PCI)
2233  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
2234 
2235  drmFreeDevice(&drm_dev_info);
2236 
2237  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2238  }
2239 #endif
2240 #if CONFIG_CUDA
2241  case AV_HWDEVICE_TYPE_CUDA: {
2242  AVHWDeviceContext *cuda_cu = src_ctx;
2243  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2244  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2245  CudaFunctions *cu = cu_internal->cuda_dl;
2246 
2247  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2248  cu_internal->cuda_device));
2249  if (ret < 0) {
2250  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2251  return AVERROR_EXTERNAL;
2252  }
2253 
2254  dev_select.has_uuid = 1;
2255 
2256  /*
2257  * CUDA is not able to import multiplane images, so always derive a
2258  * Vulkan device with multiplane disabled.
2259  */
2260  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2261  }
2262 #endif
2263  default:
2264  return AVERROR(ENOSYS);
2265  }
2266 }
2267 
2268 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2269  const void *hwconfig,
2270  AVHWFramesConstraints *constraints)
2271 {
2272  int count = 0;
2273  VulkanDevicePriv *p = ctx->hwctx;
2274 
2275  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2276  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2277  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2278  VK_IMAGE_TILING_OPTIMAL,
2279  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2280  }
2281 
2282  constraints->valid_sw_formats = av_malloc_array(count + 1,
2283  sizeof(enum AVPixelFormat));
2284  if (!constraints->valid_sw_formats)
2285  return AVERROR(ENOMEM);
2286 
2287  count = 0;
2288  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2289  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2290  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2291  VK_IMAGE_TILING_OPTIMAL,
2292  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2293  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2294  }
2295  }
2296 
2297  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2298 
2299  constraints->min_width = 1;
2300  constraints->min_height = 1;
2301  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2302  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2303 
2304  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2305  if (!constraints->valid_hw_formats)
2306  return AVERROR(ENOMEM);
2307 
2308  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2309  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2310 
2311  return 0;
2312 }
2313 
2314 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2315  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2316  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2317 {
2318  VkResult ret;
2319  int index = -1;
2320  VulkanDevicePriv *p = ctx->hwctx;
2321  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2322  AVVulkanDeviceContext *dev_hwctx = &p->p;
2323  VkMemoryAllocateInfo alloc_info = {
2324  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2325  .pNext = alloc_extension,
2326  .allocationSize = req->size,
2327  };
2328 
2329  /* The vulkan spec requires memory types to be sorted in the "optimal"
2330  * order, so the first matching type we find will be the best/fastest one */
2331  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2332  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2333 
2334  /* The memory type must be supported by the requirements (bitfield) */
2335  if (!(req->memoryTypeBits & (1 << i)))
2336  continue;
2337 
2338  /* The memory type flags must include our properties */
2339  if ((type->propertyFlags & req_flags) != req_flags)
2340  continue;
2341 
2342  /* The memory type must be large enough */
2343  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2344  continue;
2345 
2346  /* Found a suitable memory type */
2347  index = i;
2348  break;
2349  }
2350 
2351  if (index < 0) {
2352  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2353  req_flags);
2354  return AVERROR(EINVAL);
2355  }
2356 
2357  alloc_info.memoryTypeIndex = index;
2358 
2359  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2360  dev_hwctx->alloc, mem);
2361  if (ret != VK_SUCCESS) {
2362  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2363  ff_vk_ret2str(ret));
2364  return AVERROR(ENOMEM);
2365  }
2366 
2367  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2368 
2369  return 0;
2370 }
2371 
2372 static void vulkan_free_internal(AVVkFrame *f)
2373 {
2374  av_unused AVVkFrameInternal *internal = f->internal;
2375 
2376 #if CONFIG_CUDA
2377  if (internal->cuda_fc_ref) {
2378  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2379  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2380  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2381  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2382  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2383  CudaFunctions *cu = cu_internal->cuda_dl;
2384 
2385  for (int i = 0; i < planes; i++) {
2386  if (internal->cu_sem[i])
2387  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2388  if (internal->cu_mma[i])
2389  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2390  if (internal->ext_mem[i])
2391  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2392 #ifdef _WIN32
2393  if (internal->ext_sem_handle[i])
2394  CloseHandle(internal->ext_sem_handle[i]);
2395  if (internal->ext_mem_handle[i])
2396  CloseHandle(internal->ext_mem_handle[i]);
2397 #endif
2398  }
2399 
2400  av_buffer_unref(&internal->cuda_fc_ref);
2401  }
2402 #endif
2403 
2404  pthread_mutex_destroy(&internal->update_mutex);
2405  av_freep(&f->internal);
2406 }
2407 
2408 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2409 {
2410  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2411  AVVulkanDeviceContext *hwctx = &p->p;
2412  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2413  int nb_images = ff_vk_count_images(f);
2414  int nb_sems = 0;
2415 
2416  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2417  nb_sems++;
2418 
2419  if (nb_sems) {
2420  VkSemaphoreWaitInfo sem_wait = {
2421  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2422  .flags = 0x0,
2423  .pSemaphores = f->sem,
2424  .pValues = f->sem_value,
2425  .semaphoreCount = nb_sems,
2426  };
2427 
2428  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2429  }
2430 
2431  vulkan_free_internal(f);
2432 
2433  for (int i = 0; i < nb_images; i++) {
2434  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2435  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2436  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2437  }
2438 
2439  av_free(f);
2440 }
2441 
2442 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2443 {
2444  vulkan_frame_free(opaque, (AVVkFrame*)data);
2445 }
2446 
2447 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2448  void *alloc_pnext, size_t alloc_pnext_stride)
2449 {
2450  int img_cnt = 0, err;
2451  VkResult ret;
2452  AVHWDeviceContext *ctx = hwfc->device_ctx;
2453  VulkanDevicePriv *p = ctx->hwctx;
2454  AVVulkanDeviceContext *hwctx = &p->p;
2455  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2456  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2457 
2458  while (f->img[img_cnt]) {
2459  int use_ded_mem;
2460  VkImageMemoryRequirementsInfo2 req_desc = {
2461  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2462  .image = f->img[img_cnt],
2463  };
2464  VkMemoryDedicatedAllocateInfo ded_alloc = {
2465  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2466  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2467  };
2468  VkMemoryDedicatedRequirements ded_req = {
2469  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2470  };
2471  VkMemoryRequirements2 req = {
2472  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2473  .pNext = &ded_req,
2474  };
2475 
2476  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2477 
2478  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2479  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2480  p->props.properties.limits.minMemoryMapAlignment);
2481 
2482  /* In case the implementation prefers/requires dedicated allocation */
2483  use_ded_mem = ded_req.prefersDedicatedAllocation |
2484  ded_req.requiresDedicatedAllocation;
2485  if (use_ded_mem)
2486  ded_alloc.image = f->img[img_cnt];
2487 
2488  /* Allocate memory */
2489  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2490  f->tiling == VK_IMAGE_TILING_LINEAR ?
2491  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2492  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2493  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2494  &f->flags, &f->mem[img_cnt])))
2495  return err;
2496 
2497  f->size[img_cnt] = req.memoryRequirements.size;
2498  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2499  bind_info[img_cnt].image = f->img[img_cnt];
2500  bind_info[img_cnt].memory = f->mem[img_cnt];
2501 
2502  img_cnt++;
2503  }
2504 
2505  /* Bind the allocated memory to the images */
2506  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2507  if (ret != VK_SUCCESS) {
2508  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2509  ff_vk_ret2str(ret));
2510  return AVERROR_EXTERNAL;
2511  }
2512 
2513  return 0;
2514 }
2515 
2516 enum PrepMode {
2517  PREP_MODE_GENERAL,
2518  PREP_MODE_WRITE,
2519  PREP_MODE_EXTERNAL_EXPORT,
2520  PREP_MODE_EXTERNAL_IMPORT,
2521  PREP_MODE_DECODING_DST,
2522  PREP_MODE_DECODING_DPB,
2523  PREP_MODE_ENCODING_DPB,
2524 };
2525 
2526 static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout,
2527  VkAccessFlags2 *new_access)
2528 {
2529  switch (pmode) {
2530  case PREP_MODE_GENERAL:
2531  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2532  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2533  break;
2534  case PREP_MODE_WRITE:
2535  *new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2536  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2537  break;
2538  case PREP_MODE_EXTERNAL_IMPORT:
2539  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2540  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2541  break;
2542  case PREP_MODE_EXTERNAL_EXPORT:
2543  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2544  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2545  break;
2546  case PREP_MODE_DECODING_DST:
2547  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2548  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2549  break;
2550  case PREP_MODE_DECODING_DPB:
2551  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2552  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2553  break;
2554  case PREP_MODE_ENCODING_DPB:
2555  *new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2556  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2557  break;
2558  }
2559 }
2560 
2561 static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2562  AVVkFrame *frame, enum PrepMode pmode)
2563 {
2564  int err;
2565  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2566  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2567  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2568  int nb_img_bar = 0;
2569 
2570  VkImageLayout new_layout;
2571  VkAccessFlags2 new_access;
2572  switch_new_props(pmode, &new_layout, &new_access);
2573 
2574  uint32_t dst_qf = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2575  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2576  if (pmode == PREP_MODE_EXTERNAL_EXPORT) {
2577  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2578  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2579  }
2580 
2581  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2582  * free non-refcounted frames, and non-refcounted hardware frames cannot
2583  * happen anywhere outside of here. */
2584  AVBufferRef tmp_ref = {
2585  .data = (uint8_t *)hwfc,
2586  };
2587  AVFrame tmp_frame = {
2588  .data[0] = (uint8_t *)frame,
2589  .hw_frames_ctx = &tmp_ref,
2590  };
2591 
2592  VkCommandBuffer cmd_buf;
2593  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2594  cmd_buf = exec->buf;
2595  ff_vk_exec_start(&p->vkctx, exec);
2596 
2597  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2598  VK_PIPELINE_STAGE_2_NONE,
2599  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2600  if (err < 0)
2601  return err;
2602 
2603  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2604  src_stage,
2605  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2606  new_access, new_layout, dst_qf);
2607 
2608  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2609  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2610  .pImageMemoryBarriers = img_bar,
2611  .imageMemoryBarrierCount = nb_img_bar,
2612  });
2613 
2614  err = ff_vk_exec_submit(&p->vkctx, exec);
2615  if (err < 0)
2616  return err;
2617 
2618  /* We can do this because there are no real dependencies */
2619  ff_vk_exec_discard_deps(&p->vkctx, exec);
2620 
2621  return 0;
2622 }
2623 
2624 static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2625  AVVkFrame *frame, enum PrepMode pmode)
2626 {
2627  VkResult ret;
2628  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2629  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2630  VkHostImageLayoutTransitionInfoEXT layout_change[AV_NUM_DATA_POINTERS];
2631  int nb_images = ff_vk_count_images(frame);
2632 
2633  VkImageLayout new_layout;
2634  VkAccessFlags2 new_access;
2635  switch_new_props(pmode, &new_layout, &new_access);
2636 
2637  int i;
2638  for (i = 0; i < p->vkctx.host_image_props.copyDstLayoutCount; i++) {
2639  if (p->vkctx.host_image_props.pCopyDstLayouts[i] == new_layout)
2640  break;
2641  }
2642  if (i == p->vkctx.host_image_props.copyDstLayoutCount)
2643  return AVERROR(ENOTSUP);
2644 
2645  for (i = 0; i < nb_images; i++) {
2646  layout_change[i] = (VkHostImageLayoutTransitionInfoEXT) {
2647  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
2648  .image = frame->img[i],
2649  .oldLayout = frame->layout[i],
2650  .newLayout = new_layout,
2651  .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
2652  .subresourceRange.layerCount = 1,
2653  .subresourceRange.levelCount = 1,
2654  };
2655  frame->layout[i] = new_layout;
2656  }
2657 
2658  ret = vk->TransitionImageLayoutEXT(p->vkctx.hwctx->act_dev,
2659  nb_images, layout_change);
2660  if (ret != VK_SUCCESS) {
2661  av_log(hwfc, AV_LOG_ERROR, "Unable to prepare frame: %s\n",
2662  ff_vk_ret2str(ret));
2663  return AVERROR_EXTERNAL;
2664  }
2665 
2666  return 0;
2667 }
2668 
2669 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2670  AVVkFrame *frame, enum PrepMode pmode)
2671 {
2672  int err = 0;
2673  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2674  if (hwfc_vk->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
2675  (pmode != PREP_MODE_EXTERNAL_EXPORT) &&
2676  (pmode != PREP_MODE_EXTERNAL_IMPORT))
2677  err = switch_layout_host(hwfc, ectx, frame, pmode);
2678 
2679  if (err != AVERROR(ENOTSUP))
2680  return err;
2681 
2682  return switch_layout(hwfc, ectx, frame, pmode);
2683 }
2684 
2685 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2686  int frame_w, int frame_h, int plane)
2687 {
2688  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2689 
2690  /* Currently always true unless gray + alpha support is added */
2691  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2692  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2693  *w = frame_w;
2694  *h = frame_h;
2695  return;
2696  }
2697 
2698  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2699  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2700 }
2701 
2702 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2703  VkImageTiling tiling, VkImageUsageFlagBits usage,
2704  VkImageCreateFlags flags, int nb_layers,
2705  void *create_pnext)
2706 {
2707  int err;
2708  VkResult ret;
2709  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2710  AVHWDeviceContext *ctx = hwfc->device_ctx;
2711  VulkanDevicePriv *p = ctx->hwctx;
2712  AVVulkanDeviceContext *hwctx = &p->p;
2713  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2714  AVVkFrame *f;
2715 
2716  VkSemaphoreTypeCreateInfo sem_type_info = {
2717  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2718  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2719  .initialValue = 0,
2720  };
2721  VkSemaphoreCreateInfo sem_spawn = {
2722  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2723  .pNext = &sem_type_info,
2724  };
2725 
2726  VkExportSemaphoreCreateInfo ext_sem_info_opaque = {
2727  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2728 #ifdef _WIN32
2729  .handleTypes = IsWindows8OrGreater()
2730  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2731  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2732 #else
2733  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2734 #endif
2735  };
2736 
2737  /* Check if exporting is supported before chaining any structs */
2738  if (p->ext_sem_props_opaque.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) {
2739  if (p->vkctx.extensions & (FF_VK_EXT_EXTERNAL_WIN32_SEM | FF_VK_EXT_EXTERNAL_FD_SEM))
2740  ff_vk_link_struct(&sem_type_info, &ext_sem_info_opaque);
2741  }
2742 
2743  f = av_vk_frame_alloc();
2744  if (!f) {
2745  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2746  return AVERROR(ENOMEM);
2747  }
2748 
2749  // TODO: check width and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2750 
2751  /* Create the images */
2752  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2753  VkImageCreateInfo create_info = {
2754  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2755  .pNext = create_pnext,
2756  .imageType = VK_IMAGE_TYPE_2D,
2757  .format = hwfc_vk->format[i],
2758  .extent.depth = 1,
2759  .mipLevels = 1,
2760  .arrayLayers = nb_layers,
2761  .flags = flags,
2762  .tiling = tiling,
2763  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2764  .usage = usage,
2765  .samples = VK_SAMPLE_COUNT_1_BIT,
2766  .pQueueFamilyIndices = p->img_qfs,
2767  .queueFamilyIndexCount = p->nb_img_qfs,
2768  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2769  VK_SHARING_MODE_EXCLUSIVE,
2770  };
2771 
2772  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2773  hwfc->sw_format, hwfc->width, hwfc->height, i);
2774 
2775  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2776  hwctx->alloc, &f->img[i]);
2777  if (ret != VK_SUCCESS) {
2778  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2779  ff_vk_ret2str(ret));
2780  err = AVERROR(EINVAL);
2781  goto fail;
2782  }
2783 
2784  /* Create semaphore */
2785  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2786  hwctx->alloc, &f->sem[i]);
2787  if (ret != VK_SUCCESS) {
2788  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2789  ff_vk_ret2str(ret));
2790  err = AVERROR_EXTERNAL;
2791  goto fail;
2792  }
2793 
2794  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2795  f->layout[i] = create_info.initialLayout;
2796  f->access[i] = 0x0;
2797  f->sem_value[i] = 0;
2798  }
2799 
2800  f->flags = 0x0;
2801  f->tiling = tiling;
2802 
2803  *frame = f;
2804  return 0;
2805 
2806 fail:
2807  vulkan_frame_free(hwfc, f);
2808  return err;
2809 }
2810 
2811 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2812 static void try_export_flags(AVHWFramesContext *hwfc,
2813  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2814  VkExternalMemoryHandleTypeFlags *iexp,
2815  VkExternalMemoryHandleTypeFlagBits exp)
2816 {
2817  VkResult ret;
2818  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2819  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2820  AVVulkanDeviceContext *dev_hwctx = &p->p;
2821  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2822 
2823  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2824  ff_vk_find_struct(hwctx->create_pnext,
2825  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2826  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2827  int nb_mods;
2828 
2829  VkExternalImageFormatProperties eprops = {
2830  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2831  };
2832  VkImageFormatProperties2 props = {
2833  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2834  .pNext = &eprops,
2835  };
2836  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2837  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2838  .pNext = NULL,
2839  .pQueueFamilyIndices = p->img_qfs,
2840  .queueFamilyIndexCount = p->nb_img_qfs,
2841  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2842  VK_SHARING_MODE_EXCLUSIVE,
2843  };
2844  VkPhysicalDeviceExternalImageFormatInfo enext = {
2845  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2846  .handleType = exp,
2847  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2848  };
2849  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2850  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2851  .pNext = !exp ? NULL : &enext,
2852  .format = vk_find_format_entry(hwfc->sw_format)->vkf,
2853  .type = VK_IMAGE_TYPE_2D,
2854  .tiling = hwctx->tiling,
2855  .usage = hwctx->usage,
2856  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2857  };
2858 
2859  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2860  for (int i = 0; i < nb_mods; i++) {
2861  if (has_mods)
2862  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2863 
2864  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2865  &pinfo, &props);
2866 
2867  if (ret == VK_SUCCESS) {
2868  *iexp |= exp;
2869  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2870  }
2871  }
2872 }
2873 
2874 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2875 {
2876  int err;
2877  AVVkFrame *f;
2878  AVBufferRef *avbuf = NULL;
2879  AVHWFramesContext *hwfc = opaque;
2880  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2881  VulkanFramesPriv *fp = hwfc->hwctx;
2882  AVVulkanFramesContext *hwctx = &fp->p;
2883  VkExternalMemoryHandleTypeFlags e = 0x0;
2884  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2885 
2886  VkExternalMemoryImageCreateInfo eiinfo = {
2887  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2888  .pNext = hwctx->create_pnext,
2889  };
2890 
2891 #ifdef _WIN32
2892  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2893  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2894  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2895  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2896 #else
2897  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2898  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2899  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2900 
2901  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_DMABUF_MEMORY &&
2902  hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2903  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2904  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
2905 #endif
2906 
2907  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2908  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2909  eminfo[i].pNext = hwctx->alloc_pnext[i];
2910  eminfo[i].handleTypes = e;
2911  }
2912 
2913  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2914  hwctx->nb_layers,
2915  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2916  if (err)
2917  return NULL;
2918 
2919  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2920  if (err)
2921  goto fail;
2922 
2923  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2924  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2925  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2926  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2927  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2928  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2929  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2930  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2931  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2932  else
2933  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2934  if (err)
2935  goto fail;
2936 
2937  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2938  vulkan_frame_free_cb, hwfc, 0);
2939  if (!avbuf)
2940  goto fail;
2941 
2942  return avbuf;
2943 
2944 fail:
2945  vulkan_frame_free(hwfc, f);
2946  return NULL;
2947 }
2948 
2949 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2950 {
2951  pthread_mutex_lock(&vkf->internal->update_mutex);
2952 }
2953 
2954 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2955 {
2956  pthread_mutex_unlock(&vkf->internal->update_mutex);
2957 }
2958 
2959 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2960 {
2961  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2962  VulkanFramesPriv *fp = hwfc->hwctx;
2963 
2964  if (fp->modifier_info) {
2965  if (fp->modifier_info->pDrmFormatModifiers)
2966  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2967  av_freep(&fp->modifier_info);
2968  }
2969 
2970  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2971  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2972  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2973 
2974  av_buffer_pool_uninit(&fp->tmp);
2975 }
2976 
2977 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2978 {
2979  int err;
2980  AVVkFrame *f;
2981  VulkanFramesPriv *fp = hwfc->hwctx;
2982  AVVulkanFramesContext *hwctx = &fp->p;
2983  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2984  AVVulkanDeviceContext *dev_hwctx = &p->p;
2985  VkImageUsageFlags supported_usage;
2986  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2987  const struct FFVkFormatEntry *fmt;
2988  int disable_multiplane = p->disable_multiplane ||
2989  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2990  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2991  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2992  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2993 
2994  /* Defaults */
2995  if (!hwctx->nb_layers)
2996  hwctx->nb_layers = 1;
2997 
2998  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
2999  if (p->use_linear_images &&
3000  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
3001  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
3002 
3003 
3004  fmt = vk_find_format_entry(hwfc->sw_format);
3005  if (!fmt) {
3006  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
3007  av_get_pix_fmt_name(hwfc->sw_format));
3008  return AVERROR(ENOTSUP);
3009  }
3010 
3011  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
3012  if (hwctx->format[0] != fmt->vkf) {
3013  for (int i = 0; i < fmt->nb_images_fallback; i++) {
3014  if (hwctx->format[i] != fmt->fallback[i]) {
3015  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
3016  "for the current sw_format %s!\n",
3017  av_get_pix_fmt_name(hwfc->sw_format));
3018  return AVERROR(EINVAL);
3019  }
3020  }
3021  }
3022 
3023  /* Check if the sw_format itself is supported */
3024  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3025  hwctx->tiling, NULL,
3026  NULL, NULL, &supported_usage, 0,
3027  !hwctx->usage ||
3028  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3029  if (err < 0) {
3030  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
3031  av_get_pix_fmt_name(hwfc->sw_format));
3032  return AVERROR(EINVAL);
3033  }
3034  } else {
3035  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3036  hwctx->tiling, hwctx->format, NULL,
3037  NULL, &supported_usage,
3038  disable_multiplane,
3039  !hwctx->usage ||
3040  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3041  if (err < 0)
3042  return err;
3043  }
3044 
3045  /* Lone DPB images do not need additional flags. */
3046  if (!is_lone_dpb) {
3047  /* Image usage flags */
3048  hwctx->usage |= supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
3049  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3050  VK_IMAGE_USAGE_STORAGE_BIT |
3051  VK_IMAGE_USAGE_SAMPLED_BIT);
3052 
3053  if (p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY &&
3054  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) &&
3055  !(p->dprops.driverID == VK_DRIVER_ID_MOLTENVK))
3056  hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
3057 
3058  /* Enables encoding of images, if supported by format and extensions */
3059  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3060  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3061  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
3062  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
3063 
3064  /* Image creation flags.
3065  * Only fill them in automatically if the image is not going to be used as
3066  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
3067  if (!hwctx->img_flags) {
3068  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
3069  VK_IMAGE_USAGE_STORAGE_BIT);
3070  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
3071  if (sampleable) {
3072  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
3073  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
3074  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
3075  }
3076  }
3077  }
3078 
3079  /* If the image has an ENCODE_SRC usage, and the maintenance1
3080  * extension is supported, check if it has a profile list.
3081  * If there's no profile list, or it has no encode operations,
3082  * then allow creating the image with no specific profile. */
3083  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3084  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3085  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1)) {
3086  const VkVideoProfileListInfoKHR *pl;
3087  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
3088  if (!pl) {
3089  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3090  } else {
3091  uint32_t i;
3092  for (i = 0; i < pl->profileCount; i++) {
3093  /* Video ops start at exactly 0x00010000 */
3094  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
3095  break;
3096  }
3097  if (i == pl->profileCount)
3098  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3099  }
3100  }
3101 
3102  if (!hwctx->lock_frame)
3103  hwctx->lock_frame = lock_frame;
3104 
3105  if (!hwctx->unlock_frame)
3106  hwctx->unlock_frame = unlock_frame;
3107 
3108  err = ff_vk_exec_pool_init(&p->vkctx, p->compute_qf, &fp->compute_exec,
3109  p->compute_qf->num, 0, 0, 0, NULL);
3110  if (err)
3111  return err;
3112 
3113  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->upload_exec,
3114  p->transfer_qf->num*2, 0, 0, 0, NULL);
3115  if (err)
3116  return err;
3117 
3118  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->download_exec,
3119  p->transfer_qf->num, 0, 0, 0, NULL);
3120  if (err)
3121  return err;
3122 
3123  /* Test to see if allocation will fail */
3124  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
3125  hwctx->nb_layers, hwctx->create_pnext);
3126  if (err)
3127  return err;
3128 
3129  /* Collect `VkDrmFormatModifierPropertiesEXT` for each plane. Required for DRM export. */
3130  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS && hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
3131  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3132  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3133  };
3134  err = vk->GetImageDrmFormatModifierPropertiesEXT(dev_hwctx->act_dev, f->img[0],
3135  &drm_mod);
3136  if (err != VK_SUCCESS) {
3137  av_log(hwfc, AV_LOG_ERROR, "Failed to get image DRM format modifier properties");
3138  vulkan_frame_free(hwfc, f);
3139  return AVERROR_EXTERNAL;
3140  }
3141  for (int i = 0; i < fmt->vk_planes; ++i) {
3142  VkDrmFormatModifierPropertiesListEXT modp;
3143  VkFormatProperties2 fmtp;
3144  VkDrmFormatModifierPropertiesEXT *mod_props = NULL;
3145 
3146  modp = (VkDrmFormatModifierPropertiesListEXT) {
3147  .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
3148  };
3149  fmtp = (VkFormatProperties2) {
3150  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
3151  .pNext = &modp,
3152  };
3153 
3154  /* query drmFormatModifierCount by keeping pDrmFormatModifierProperties NULL */
3155  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3156 
3157  modp.pDrmFormatModifierProperties =
3158  av_calloc(modp.drmFormatModifierCount, sizeof(*modp.pDrmFormatModifierProperties));
3159  if (!modp.pDrmFormatModifierProperties) {
3160  vulkan_frame_free(hwfc, f);
3161  return AVERROR(ENOMEM);
3162  }
3163  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3164 
3165  for (uint32_t i = 0; i < modp.drmFormatModifierCount; ++i) {
3166  VkDrmFormatModifierPropertiesEXT *m = &modp.pDrmFormatModifierProperties[i];
3167  if (m->drmFormatModifier == drm_mod.drmFormatModifier) {
3168  mod_props = m;
3169  break;
3170  }
3171  }
3172 
3173  if (mod_props == NULL) {
3174  av_log(hwfc, AV_LOG_ERROR, "No DRM format modifier properties found for modifier 0x%016"PRIx64"\n",
3175  drm_mod.drmFormatModifier);
3176  av_free(modp.pDrmFormatModifierProperties);
3177  vulkan_frame_free(hwfc, f);
3178  return AVERROR_EXTERNAL;
3179  }
3180 
3181  fp->drm_format_modifier_properties[i] = *mod_props;
3182  av_free(modp.pDrmFormatModifierProperties);
3183  }
3184  }
3185 
3186  vulkan_frame_free(hwfc, f);
3187 
3188  /* If user did not specify a pool, hwfc->pool will be set to the internal one
3189  * in hwcontext.c just after this gets called */
3190  if (!hwfc->pool) {
3191  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
3192  hwfc, vulkan_pool_alloc,
3193  NULL);
3194  if (!ffhwframesctx(hwfc)->pool_internal)
3195  return AVERROR(ENOMEM);
3196  }
3197 
3198  return 0;
3199 }
3200 
3201 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
3202 {
3203  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
3204  if (!frame->buf[0])
3205  return AVERROR(ENOMEM);
3206 
3207  frame->data[0] = frame->buf[0]->data;
3208  frame->format = AV_PIX_FMT_VULKAN;
3209  frame->width = hwfc->width;
3210  frame->height = hwfc->height;
3211 
3212  return 0;
3213 }
3214 
3215 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
3216  enum AVHWFrameTransferDirection dir,
3217  enum AVPixelFormat **formats)
3218 {
3219  enum AVPixelFormat *fmts;
3220  int n = 2;
3221 
3222 #if CONFIG_CUDA
3223  n++;
3224 #endif
3225  fmts = av_malloc_array(n, sizeof(*fmts));
3226  if (!fmts)
3227  return AVERROR(ENOMEM);
3228 
3229  n = 0;
3230  fmts[n++] = hwfc->sw_format;
3231 #if CONFIG_CUDA
3232  fmts[n++] = AV_PIX_FMT_CUDA;
3233 #endif
3234  fmts[n++] = AV_PIX_FMT_NONE;
3235 
3236  *formats = fmts;
3237  return 0;
3238 }
3239 
3240 #if CONFIG_LIBDRM
3241 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3242 {
3243  vulkan_frame_free(hwfc, hwmap->priv);
3244 }
3245 
3246 static const struct {
3247  uint32_t drm_fourcc;
3248  VkFormat vk_format;
3249 } vulkan_drm_format_map[] = {
3250  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
3251  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
3252  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
3253  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
3254  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
3255  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
3256  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3257  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3258  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3259  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3260  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3261  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3262  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3263  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3264 
3265  // All these DRM_FORMATs were added in the same libdrm commit.
3266 #ifdef DRM_FORMAT_XYUV8888
3267  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
3268  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
3269  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
3270  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
3271 #endif
3272 };
3273 
3274 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
3275 {
3276  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3277  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
3278  return vulkan_drm_format_map[i].vk_format;
3279  return VK_FORMAT_UNDEFINED;
3280 }
3281 
3282 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
3283  const AVFrame *src, int flags)
3284 {
3285  int err = 0;
3286  VkResult ret;
3287  AVVkFrame *f;
3288  int bind_counts = 0;
3289  AVHWDeviceContext *ctx = hwfc->device_ctx;
3290  VulkanDevicePriv *p = ctx->hwctx;
3291  AVVulkanDeviceContext *hwctx = &p->p;
3292  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3293  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3294  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
3295  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
3296 
3297  for (int i = 0; i < desc->nb_layers; i++) {
3298  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
3299  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
3300  desc->layers[i].format);
3301  return AVERROR(EINVAL);
3302  }
3303  }
3304 
3305  if (!(f = av_vk_frame_alloc())) {
3306  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
3307  err = AVERROR(ENOMEM);
3308  goto fail;
3309  }
3310 
3311  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
3312 
3313  for (int i = 0; i < desc->nb_layers; i++) {
3314  const int planes = desc->layers[i].nb_planes;
3315 
3316  /* Semaphore */
3317  VkSemaphoreTypeCreateInfo sem_type_info = {
3318  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3319  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
3320  .initialValue = 0,
3321  };
3322  VkSemaphoreCreateInfo sem_spawn = {
3323  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3324  .pNext = &sem_type_info,
3325  };
3326 
3327  /* Image creation */
3328  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
3329  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
3330  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
3331  .drmFormatModifier = desc->objects[0].format_modifier,
3332  .drmFormatModifierPlaneCount = planes,
3333  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
3334  };
3335  VkExternalMemoryImageCreateInfo ext_img_spec = {
3336  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
3337  .pNext = &ext_img_mod_spec,
3338  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3339  };
3340  VkImageCreateInfo create_info = {
3341  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
3342  .pNext = &ext_img_spec,
3343  .imageType = VK_IMAGE_TYPE_2D,
3344  .format = drm_to_vulkan_fmt(desc->layers[i].format),
3345  .extent.depth = 1,
3346  .mipLevels = 1,
3347  .arrayLayers = 1,
3348  .flags = 0x0,
3349  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
3350  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
3351  .usage = 0x0, /* filled in below */
3352  .samples = VK_SAMPLE_COUNT_1_BIT,
3353  .pQueueFamilyIndices = p->img_qfs,
3354  .queueFamilyIndexCount = p->nb_img_qfs,
3355  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
3356  VK_SHARING_MODE_EXCLUSIVE,
3357  };
3358 
3359  /* Image format verification */
3360  VkExternalImageFormatProperties ext_props = {
3361  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
3362  };
3363  VkImageFormatProperties2 props_ret = {
3364  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
3365  .pNext = &ext_props,
3366  };
3367  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
3368  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
3369  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
3370  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
3371  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
3372  .sharingMode = create_info.sharingMode,
3373  };
3374  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
3375  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3376  .pNext = &props_drm_mod,
3377  .handleType = ext_img_spec.handleTypes,
3378  };
3379  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3380 
3381  if (flags & AV_HWFRAME_MAP_READ)
3382  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3383  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3384  if (flags & AV_HWFRAME_MAP_WRITE)
3385  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3386  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3387 
3388  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3389  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3390  .pNext = &props_ext,
3391  .format = create_info.format,
3392  .type = create_info.imageType,
3393  .tiling = create_info.tiling,
3394  .usage = create_info.usage,
3395  .flags = create_info.flags,
3396  };
3397 
3398  /* Check if importing is possible for this combination of parameters */
3399  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3400  &fmt_props, &props_ret);
3401  if (ret != VK_SUCCESS) {
3402  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3403  ff_vk_ret2str(ret));
3404  err = AVERROR_EXTERNAL;
3405  goto fail;
3406  }
3407 
3408  /* Set the image width/height */
3409  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3410  hwfc->sw_format, src->width, src->height, i);
3411 
3412  /* Set the subresource layout based on the layer properties */
3413  for (int j = 0; j < planes; j++) {
3414  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3415  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3416  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3417  ext_img_layouts[j].arrayPitch = 0;
3418  ext_img_layouts[j].depthPitch = 0;
3419  }
3420 
3421  /* Create image */
3422  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3423  hwctx->alloc, &f->img[i]);
3424  if (ret != VK_SUCCESS) {
3425  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3426  ff_vk_ret2str(ret));
3427  err = AVERROR(EINVAL);
3428  goto fail;
3429  }
3430 
3431  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3432  hwctx->alloc, &f->sem[i]);
3433  if (ret != VK_SUCCESS) {
3434  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3435  ff_vk_ret2str(ret));
3436  err = AVERROR_EXTERNAL;
3437  goto fail;
3438  }
3439 
3440  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3441  f->layout[i] = create_info.initialLayout;
3442  f->access[i] = 0x0;
3443  f->sem_value[i] = 0;
3444  }
3445 
3446  for (int i = 0; i < desc->nb_layers; i++) {
3447  /* Memory requirements */
3448  VkImageMemoryRequirementsInfo2 req_desc = {
3449  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3450  .image = f->img[i],
3451  };
3452  VkMemoryDedicatedRequirements ded_req = {
3453  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3454  };
3455  VkMemoryRequirements2 req2 = {
3456  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3457  .pNext = &ded_req,
3458  };
3459 
3460  /* Allocation/importing */
3461  VkMemoryFdPropertiesKHR fdmp = {
3462  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3463  };
3464  /* This assumes that a layer will never be constructed from multiple
3465  * objects. If that was to happen in the real world, this code would
3466  * need to import each plane separately.
3467  */
3468  VkImportMemoryFdInfoKHR idesc = {
3469  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3470  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3471  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3472  };
3473  VkMemoryDedicatedAllocateInfo ded_alloc = {
3474  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3475  .pNext = &idesc,
3476  .image = req_desc.image,
3477  };
3478 
3479  /* Get object properties */
3480  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3481  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3482  idesc.fd, &fdmp);
3483  if (ret != VK_SUCCESS) {
3484  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3485  ff_vk_ret2str(ret));
3486  err = AVERROR_EXTERNAL;
3487  close(idesc.fd);
3488  goto fail;
3489  }
3490 
3491  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3492 
3493  /* Only a single bit must be set, not a range, and it must match */
3494  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3495 
3496  err = alloc_mem(ctx, &req2.memoryRequirements,
3497  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3498  (ded_req.prefersDedicatedAllocation ||
3499  ded_req.requiresDedicatedAllocation) ?
3500  &ded_alloc : ded_alloc.pNext,
3501  &f->flags, &f->mem[i]);
3502  if (err) {
3503  close(idesc.fd);
3504  return err;
3505  }
3506 
3507  f->size[i] = req2.memoryRequirements.size;
3508  }
3509 
3510  for (int i = 0; i < desc->nb_layers; i++) {
3511  const int planes = desc->layers[i].nb_planes;
3512  for (int j = 0; j < planes; j++) {
3513  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3514  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3515  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3516 
3517  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3518  plane_info[bind_counts].pNext = NULL;
3519  plane_info[bind_counts].planeAspect = aspect;
3520 
3521  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3522  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3523  bind_info[bind_counts].image = f->img[i];
3524  bind_info[bind_counts].memory = f->mem[i];
3525 
3526  /* Offset is already signalled via pPlaneLayouts above */
3527  bind_info[bind_counts].memoryOffset = 0;
3528 
3529  bind_counts++;
3530  }
3531  }
3532 
3533  /* Bind the allocated memory to the images */
3534  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3535  if (ret != VK_SUCCESS) {
3536  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3537  ff_vk_ret2str(ret));
3538  err = AVERROR_EXTERNAL;
3539  goto fail;
3540  }
3541 
3542  *frame = f;
3543 
3544  return 0;
3545 
3546 fail:
3547  vulkan_frame_free(hwfc, f);
3548 
3549  return err;
3550 }
3551 
3552 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3553  const AVDRMFrameDescriptor *desc, int flags)
3554 {
3555  int err;
3556  VkResult ret;
3557  AVHWDeviceContext *ctx = hwfc->device_ctx;
3558  VulkanDevicePriv *p = ctx->hwctx;
3559  VulkanFramesPriv *fp = hwfc->hwctx;
3560  AVVulkanDeviceContext *hwctx = &p->p;
3561  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3562 
3563 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3564  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3565  VkCommandBuffer cmd_buf;
3566  FFVkExecContext *exec;
3567  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3568  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3569  int nb_img_bar = 0;
3570 
3571  for (int i = 0; i < desc->nb_objects; i++) {
3572  VkSemaphoreTypeCreateInfo sem_type_info = {
3573  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3574  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3575  };
3576  VkSemaphoreCreateInfo sem_spawn = {
3577  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3578  .pNext = &sem_type_info,
3579  };
3580  VkImportSemaphoreFdInfoKHR import_info;
3581  struct dma_buf_export_sync_file implicit_fd_info = {
3582  .flags = DMA_BUF_SYNC_READ,
3583  .fd = -1,
3584  };
3585 
3586  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3587  &implicit_fd_info)) {
3588  err = AVERROR(errno);
3589  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3590  av_err2str(err));
3591  for (; i >= 0; i--)
3592  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3593  return err;
3594  }
3595 
3596  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3597  hwctx->alloc, &drm_sync_sem[i]);
3598  if (ret != VK_SUCCESS) {
3599  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3600  ff_vk_ret2str(ret));
3601  err = AVERROR_EXTERNAL;
3602  for (; i >= 0; i--)
3603  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3604  return err;
3605  }
3606 
3607  import_info = (VkImportSemaphoreFdInfoKHR) {
3608  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3609  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3610  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3611  .semaphore = drm_sync_sem[i],
3612  .fd = implicit_fd_info.fd,
3613  };
3614 
3615  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3616  if (ret != VK_SUCCESS) {
3617  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3618  ff_vk_ret2str(ret));
3619  err = AVERROR_EXTERNAL;
3620  for (; i >= 0; i--)
3621  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3622  return err;
3623  }
3624  }
3625 
3626  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3627  cmd_buf = exec->buf;
3628 
3629  ff_vk_exec_start(&p->vkctx, exec);
3630 
3631  /* Ownership of semaphores is passed */
3632  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3633  drm_sync_sem, desc->nb_objects,
3634  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3635  if (err < 0)
3636  return err;
3637 
3638  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3639  VK_PIPELINE_STAGE_2_NONE,
3640  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3641  if (err < 0)
3642  return err;
3643 
3644  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3645  VK_PIPELINE_STAGE_2_NONE,
3646  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3647  ((flags & AV_HWFRAME_MAP_READ) ?
3648  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3649  ((flags & AV_HWFRAME_MAP_WRITE) ?
3650  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3651  VK_IMAGE_LAYOUT_GENERAL,
3652  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
3653 
3654  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3655  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3656  .pImageMemoryBarriers = img_bar,
3657  .imageMemoryBarrierCount = nb_img_bar,
3658  });
3659 
3660  err = ff_vk_exec_submit(&p->vkctx, exec);
3661  if (err < 0)
3662  return err;
3663  } else
3664 #endif
3665  {
3666  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3667  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3668  "image may be corrupted.\n");
3669  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3670  if (err)
3671  return err;
3672  }
3673 
3674  return 0;
3675 }
3676 
3677 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3678  const AVFrame *src, int flags)
3679 {
3680  int err = 0;
3681  AVVkFrame *f;
3682  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3683 
3684  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3685  return err;
3686 
3687  /* The unmapping function will free this */
3688  dst->data[0] = (uint8_t *)f;
3689  dst->width = src->width;
3690  dst->height = src->height;
3691 
3692  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3693  &vulkan_unmap_from_drm, f);
3694  if (err < 0)
3695  goto fail;
3696 
3697  err = vulkan_map_from_drm_frame_sync(hwfc, dst, desc, flags);
3698  if (err < 0)
3699  return err;
3700 
3701  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3702 
3703  return 0;
3704 
3705 fail:
3706  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3707  dst->data[0] = NULL;
3708  return err;
3709 }
3710 
3711 #if CONFIG_VAAPI
3712 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3713  AVFrame *dst, const AVFrame *src,
3714  int flags)
3715 {
3716  int err;
3717  AVFrame *tmp = av_frame_alloc();
3718  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3719  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3720  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3721 
3722  if (!tmp)
3723  return AVERROR(ENOMEM);
3724 
3725  /* We have to sync since like the previous comment said, no semaphores */
3726  vaSyncSurface(vaapi_ctx->display, surface_id);
3727 
3728  tmp->format = AV_PIX_FMT_DRM_PRIME;
3729 
3730  err = av_hwframe_map(tmp, src, flags);
3731  if (err < 0)
3732  goto fail;
3733 
3734  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3735  if (err < 0)
3736  goto fail;
3737 
3738  err = ff_hwframe_map_replace(dst, src);
3739 
3740 fail:
3741  av_frame_free(&tmp);
3742  return err;
3743 }
3744 #endif
3745 #endif
3746 
3747 #if CONFIG_CUDA
3748 static int export_mem_to_cuda(AVHWDeviceContext *ctx,
3749  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3750  AVVkFrameInternal *dst_int, int idx,
3751  VkDeviceMemory mem, size_t size)
3752 {
3753  VkResult ret;
3754  VulkanDevicePriv *p = ctx->hwctx;
3755  AVVulkanDeviceContext *hwctx = &p->p;
3756  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3757 
3758 #ifdef _WIN32
3759  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3760  .type = IsWindows8OrGreater()
3761  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3762  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3763  .size = size,
3764  };
3765  VkMemoryGetWin32HandleInfoKHR export_info = {
3766  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3767  .memory = mem,
3768  .handleType = IsWindows8OrGreater()
3769  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3770  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3771  };
3772 
3773  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3774  &ext_desc.handle.win32.handle);
3775  if (ret != VK_SUCCESS) {
3776  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3777  ff_vk_ret2str(ret));
3778  return AVERROR_EXTERNAL;
3779  }
3780  dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle;
3781 #else
3782  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3783  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3784  .size = size,
3785  };
3786  VkMemoryGetFdInfoKHR export_info = {
3787  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3788  .memory = mem,
3789  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3790  };
3791 
3792  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3793  &ext_desc.handle.fd);
3794  if (ret != VK_SUCCESS) {
3795  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3796  ff_vk_ret2str(ret));
3797  return AVERROR_EXTERNAL;
3798  }
3799 #endif
3800 
3801  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc));
3802  if (ret < 0) {
3803 #ifndef _WIN32
3804  close(ext_desc.handle.fd);
3805 #endif
3806  return AVERROR_EXTERNAL;
3807  }
3808 
3809  return 0;
3810 }
3811 
3812 static int export_sem_to_cuda(AVHWDeviceContext *ctx,
3813  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3814  AVVkFrameInternal *dst_int, int idx,
3815  VkSemaphore sem)
3816 {
3817  VkResult ret;
3818  VulkanDevicePriv *p = ctx->hwctx;
3819  AVVulkanDeviceContext *hwctx = &p->p;
3820  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3821 
3822 #ifdef _WIN32
3823  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3824  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3825  .semaphore = sem,
3826  .handleType = IsWindows8OrGreater()
3827  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3828  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3829  };
3830  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3831  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3832  };
3833 #else
3834  VkSemaphoreGetFdInfoKHR sem_export = {
3835  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3836  .semaphore = sem,
3837  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3838  };
3839  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3840  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3841  };
3842 #endif
3843 
3844 #ifdef _WIN32
3845  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3846  &ext_sem_desc.handle.win32.handle);
3847 #else
3848  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3849  &ext_sem_desc.handle.fd);
3850 #endif
3851  if (ret != VK_SUCCESS) {
3852  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3853  ff_vk_ret2str(ret));
3854  return AVERROR_EXTERNAL;
3855  }
3856 #ifdef _WIN32
3857  dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle;
3858 #endif
3859 
3860  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx],
3861  &ext_sem_desc));
3862  if (ret < 0) {
3863 #ifndef _WIN32
3864  close(ext_sem_desc.handle.fd);
3865 #endif
3866  return AVERROR_EXTERNAL;
3867  }
3868 
3869  return 0;
3870 }
3871 
3872 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3873  AVBufferRef *cuda_hwfc,
3874  const AVFrame *frame)
3875 {
3876  int err;
3877  VkResult ret;
3878  AVVkFrame *dst_f;
3879  AVVkFrameInternal *dst_int;
3880  AVHWDeviceContext *ctx = hwfc->device_ctx;
3881  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3882  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3883  VulkanDevicePriv *p = ctx->hwctx;
3884  AVVulkanDeviceContext *hwctx = &p->p;
3885  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3886  int nb_images;
3887 
3888  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3889  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3890  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3891  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3892  CudaFunctions *cu = cu_internal->cuda_dl;
3893  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3894  CU_AD_FORMAT_UNSIGNED_INT8;
3895 
3896  dst_f = (AVVkFrame *)frame->data[0];
3897  dst_int = dst_f->internal;
3898 
3899  if (!dst_int->cuda_fc_ref) {
3900  size_t offsets[3] = { 0 };
3901 
3902  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3903  if (!dst_int->cuda_fc_ref)
3904  return AVERROR(ENOMEM);
3905 
3906  nb_images = ff_vk_count_images(dst_f);
3907  for (int i = 0; i < nb_images; i++) {
3908  err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3909  dst_f->mem[i], dst_f->size[i]);
3910  if (err < 0)
3911  goto fail;
3912 
3913  err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3914  dst_f->sem[i]);
3915  if (err < 0)
3916  goto fail;
3917  }
3918 
3919  if (nb_images != planes) {
3920  for (int i = 0; i < planes; i++) {
3921  VkImageSubresource subres = {
3922  .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT :
3923  i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3924  VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
3925  };
3926  VkSubresourceLayout layout = { 0 };
3927  vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)],
3928  &subres, &layout);
3929  offsets[i] = layout.offset;
3930  }
3931  }
3932 
3933  for (int i = 0; i < planes; i++) {
3934  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3935  .offset = offsets[i],
3936  .arrayDesc = {
3937  .Depth = 0,
3938  .Format = cufmt,
3939  .NumChannels = 1 + ((planes == 2) && i),
3940  .Flags = 0,
3941  },
3942  .numLevels = 1,
3943  };
3944  int p_w, p_h;
3945 
3946  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3947  tex_desc.arrayDesc.Width = p_w;
3948  tex_desc.arrayDesc.Height = p_h;
3949 
3950  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3951  dst_int->ext_mem[FFMIN(i, nb_images - 1)],
3952  &tex_desc));
3953  if (ret < 0) {
3954  err = AVERROR_EXTERNAL;
3955  goto fail;
3956  }
3957 
3958  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3959  dst_int->cu_mma[i], 0));
3960  if (ret < 0) {
3961  err = AVERROR_EXTERNAL;
3962  goto fail;
3963  }
3964 
3965  }
3966  }
3967 
3968  return 0;
3969 
3970 fail:
3971  vulkan_free_internal(dst_f);
3972  return err;
3973 }
3974 
3975 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3976  AVFrame *dst, const AVFrame *src)
3977 {
3978  int err;
3979  CUcontext dummy;
3980  AVVkFrame *dst_f;
3981  AVVkFrameInternal *dst_int;
3982  VulkanFramesPriv *fp = hwfc->hwctx;
3983  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3984  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3985 
3986  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3987  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3988  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3989  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3990  CudaFunctions *cu = cu_internal->cuda_dl;
3991  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3992  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3993 
3994  dst_f = (AVVkFrame *)dst->data[0];
3995 
3996  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3997  if (err < 0)
3998  return err;
3999 
4000  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4001  if (err < 0)
4002  return err;
4003 
4004  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
4005  if (err < 0) {
4006  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4007  return err;
4008  }
4009 
4010  dst_int = dst_f->internal;
4011 
4012  for (int i = 0; i < planes; i++) {
4013  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4014  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4015  }
4016 
4017  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4018  planes, cuda_dev->stream));
4019  if (err < 0)
4020  goto fail;
4021 
4022  for (int i = 0; i < planes; i++) {
4023  CUDA_MEMCPY2D cpy = {
4024  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
4025  .srcDevice = (CUdeviceptr)src->data[i],
4026  .srcPitch = src->linesize[i],
4027  .srcY = 0,
4028 
4029  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
4030  .dstArray = dst_int->cu_array[i],
4031  };
4032 
4033  int p_w, p_h;
4034  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4035 
4036  cpy.WidthInBytes = p_w * desc->comp[i].step;
4037  cpy.Height = p_h;
4038 
4039  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4040  if (err < 0)
4041  goto fail;
4042  }
4043 
4044  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4045  planes, cuda_dev->stream));
4046  if (err < 0)
4047  goto fail;
4048 
4049  for (int i = 0; i < planes; i++)
4050  dst_f->sem_value[i]++;
4051 
4052  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4053 
4054  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
4055 
4056  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4057 
4058 fail:
4059  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4060  vulkan_free_internal(dst_f);
4061  av_buffer_unref(&dst->buf[0]);
4062  return err;
4063 }
4064 #endif
4065 
4066 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
4067  const AVFrame *src, int flags)
4068 {
4069  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4070 
4071  switch (src->format) {
4072 #if CONFIG_LIBDRM
4073 #if CONFIG_VAAPI
4074  case AV_PIX_FMT_VAAPI:
4075  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4076  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
4077  else
4078  return AVERROR(ENOSYS);
4079 #endif
4080  case AV_PIX_FMT_DRM_PRIME:
4081  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4082  return vulkan_map_from_drm(hwfc, dst, src, flags);
4083  else
4084  return AVERROR(ENOSYS);
4085 #endif
4086  default:
4087  return AVERROR(ENOSYS);
4088  }
4089 }
4090 
4091 #if CONFIG_LIBDRM
4092 typedef struct VulkanDRMMapping {
4093  AVDRMFrameDescriptor drm_desc;
4094  AVVkFrame *source;
4095 } VulkanDRMMapping;
4096 
4097 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
4098 {
4099  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
4100 
4101  /* on unmap from DRM, make sure to import sync objects so that we are sync'd with any work that was
4102  * done on the buffer while exported. We don't know if who used the dmabuf did reads or writes, so protect against both */
4103  vulkan_map_from_drm_frame_sync(hwfc, hwmap->source, drm_desc, AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_WRITE);
4104 
4105  for (int i = 0; i < drm_desc->nb_objects; i++)
4106  close(drm_desc->objects[i].fd);
4107 
4108  av_free(drm_desc);
4109 }
4110 
4111 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
4112 {
4113  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
4114  if (vulkan_drm_format_map[i].vk_format == vkfmt)
4115  return vulkan_drm_format_map[i].drm_fourcc;
4116  return DRM_FORMAT_INVALID;
4117 }
4118 
4119 #define MAX_MEMORY_PLANES 4
4120 static VkImageAspectFlags plane_index_to_aspect(int plane) {
4121  if (plane == 0) return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4122  if (plane == 1) return VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
4123  if (plane == 2) return VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
4124  if (plane == 3) return VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
4125 
4126  av_assert2 (0 && "Invalid plane index");
4127  return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4128 }
4129 
4130 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
4131  const AVFrame *src, int flags)
4132 {
4133  int err = 0;
4134  VkResult ret;
4135  AVVkFrame *f = (AVVkFrame *)src->data[0];
4136  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4137  AVVulkanDeviceContext *hwctx = &p->p;
4138  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4139  VulkanFramesPriv *fp = hwfc->hwctx;
4140  const int nb_images = ff_vk_count_images(f);
4141  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
4142  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
4143  };
4144  VkSemaphoreWaitInfo wait_info = {
4145  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4146  .flags = 0x0,
4147  .semaphoreCount = nb_images,
4148  };
4149 
4150  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
4151  if (!drm_desc)
4152  return AVERROR(ENOMEM);
4153 
4154  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
4155  if (err < 0)
4156  goto end;
4157 
4158  /* Wait for the operation to finish so we can cleanly export it. */
4159  wait_info.pSemaphores = f->sem;
4160  wait_info.pValues = f->sem_value;
4161 
4162  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
4163 
4164  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
4165  if (err < 0)
4166  goto end;
4167 
4168  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
4169  &drm_mod);
4170  if (ret != VK_SUCCESS) {
4171  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
4172  err = AVERROR_EXTERNAL;
4173  goto end;
4174  }
4175 
4176  for (int i = 0; (i < nb_images) && (f->mem[i]); i++) {
4177  VkMemoryGetFdInfoKHR export_info = {
4178  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
4179  .memory = f->mem[i],
4180  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
4181  };
4182 
4183  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
4184  &drm_desc->objects[i].fd);
4185  if (ret != VK_SUCCESS) {
4186  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
4187  err = AVERROR_EXTERNAL;
4188  goto end;
4189  }
4190 
4191  drm_desc->nb_objects++;
4192  drm_desc->objects[i].size = f->size[i];
4193  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
4194  }
4195 
4196  drm_desc->nb_layers = nb_images;
4197  for (int i = 0; i < drm_desc->nb_layers; i++) {
4198  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
4199 
4200  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
4201  drm_desc->layers[i].nb_planes = fp->drm_format_modifier_properties[i].drmFormatModifierPlaneCount;
4202 
4203  if (drm_desc->layers[i].nb_planes > MAX_MEMORY_PLANES) {
4204  av_log(hwfc, AV_LOG_ERROR, "Too many memory planes for DRM format!\n");
4205  err = AVERROR_EXTERNAL;
4206  goto end;
4207  }
4208 
4209  for (int j = 0; j < drm_desc->layers[i].nb_planes; j++) {
4210  VkSubresourceLayout layout;
4211  VkImageSubresource sub = {
4212  .aspectMask = plane_index_to_aspect(j),
4213  };
4214 
4215  drm_desc->layers[i].planes[j].object_index = FFMIN(i, drm_desc->nb_objects - 1);
4216 
4217  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
4218  drm_desc->layers[i].planes[j].offset = layout.offset;
4219  drm_desc->layers[i].planes[j].pitch = layout.rowPitch;
4220  }
4221 
4222  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
4223  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
4224  err = AVERROR_PATCHWELCOME;
4225  goto end;
4226  }
4227 
4228 
4229  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
4230  continue;
4231 
4232  }
4233 
4234  dst->width = src->width;
4235  dst->height = src->height;
4236  dst->data[0] = (uint8_t *)drm_desc;
4237 
4238  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
4239 
4240  return 0;
4241 
4242 end:
4243  av_free(drm_desc);
4244  return err;
4245 }
4246 
4247 #if CONFIG_VAAPI
4248 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
4249  const AVFrame *src, int flags)
4250 {
4251  int err;
4252  AVFrame *tmp = av_frame_alloc();
4253  if (!tmp)
4254  return AVERROR(ENOMEM);
4255 
4256  tmp->format = AV_PIX_FMT_DRM_PRIME;
4257 
4258  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
4259  if (err < 0)
4260  goto fail;
4261 
4262  err = av_hwframe_map(dst, tmp, flags);
4263  if (err < 0)
4264  goto fail;
4265 
4266  err = ff_hwframe_map_replace(dst, src);
4267 
4268 fail:
4269  av_frame_free(&tmp);
4270  return err;
4271 }
4272 #endif
4273 #endif
4274 
4275 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
4276  const AVFrame *src, int flags)
4277 {
4278  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4279 
4280  switch (dst->format) {
4281 #if CONFIG_LIBDRM
4282  case AV_PIX_FMT_DRM_PRIME:
4283  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4284  return vulkan_map_to_drm(hwfc, dst, src, flags);
4285  else
4286  return AVERROR(ENOSYS);
4287 #if CONFIG_VAAPI
4288  case AV_PIX_FMT_VAAPI:
4289  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4290  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
4291  else
4292  return AVERROR(ENOSYS);
4293 #endif
4294 #endif
4295  default:
4296  break;
4297  }
4298  return AVERROR(ENOSYS);
4299 }
4300 
4301 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
4302  AVFrame *swf, VkBufferImageCopy *region,
4303  int planes, int upload)
4304 {
4305  int err;
4306  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4307  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
4308 
4309  if (upload) {
4310  for (int i = 0; i < planes; i++)
4311  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
4312  region[i].bufferRowLength,
4313  swf->data[i],
4314  swf->linesize[i],
4315  swf->linesize[i],
4316  region[i].imageExtent.height);
4317 
4318  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 1);
4319  if (err != VK_SUCCESS) {
4320  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
4321  av_err2str(err));
4322  return AVERROR_EXTERNAL;
4323  }
4324  } else {
4325  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 0);
4326  if (err != VK_SUCCESS) {
4327  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
4328  av_err2str(err));
4329  return AVERROR_EXTERNAL;
4330  }
4331 
4332  for (int i = 0; i < planes; i++)
4333  av_image_copy_plane(swf->data[i],
4334  swf->linesize[i],
4335  vkbuf->mapped_mem + region[i].bufferOffset,
4336  region[i].bufferRowLength,
4337  swf->linesize[i],
4338  region[i].imageExtent.height);
4339  }
4340 
4341  return 0;
4342 }
4343 
4344 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
4345  AVFrame *swf, VkBufferImageCopy *region, int upload)
4346 {
4347  int err;
4348  uint32_t p_w, p_h;
4349  VulkanFramesPriv *fp = hwfc->hwctx;
4350  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4351  const int planes = av_pix_fmt_count_planes(swf->format);
4352  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4353  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4354 
4355  size_t buf_offset = 0;
4356  for (int i = 0; i < planes; i++) {
4357  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4358 
4359  region[i] = (VkBufferImageCopy) {
4360  .bufferOffset = buf_offset,
4361  .bufferRowLength = FFALIGN(swf->linesize[i],
4362  p->props.properties.limits.optimalBufferCopyRowPitchAlignment),
4363  .bufferImageHeight = p_h,
4364  .imageSubresource.layerCount = 1,
4365  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4366  /* Rest of the fields adjusted/filled in later */
4367  };
4368 
4369  buf_offset += FFALIGN(p_h*region[i].bufferRowLength,
4370  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
4371  }
4372 
4373  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, buf_usage,
4374  NULL, buf_offset,
4375  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
4376  p->vkctx.host_cached_flag);
4377  if (err < 0)
4378  return err;
4379 
4380  return 0;
4381 }
4382 
4383 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
4384  AVFrame *swf, VkBufferImageCopy *region, int upload)
4385 {
4386  int err;
4387  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4388 
4389  int nb_src_bufs;
4390  const int planes = av_pix_fmt_count_planes(swf->format);
4391  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4392  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4393 
4394  /* We can't host map images with negative strides */
4395  for (int i = 0; i < planes; i++)
4396  if (swf->linesize[i] < 0)
4397  return AVERROR(EINVAL);
4398 
4399  /* Count the number of buffers in the software frame */
4400  nb_src_bufs = 0;
4401  while (swf->buf[nb_src_bufs])
4402  nb_src_bufs++;
4403 
4404  /* Single buffer contains all planes */
4405  if (nb_src_bufs == 1) {
4406  err = ff_vk_host_map_buffer(&p->vkctx, &dst[0],
4407  swf->data[0], swf->buf[0],
4408  buf_usage);
4409  if (err < 0)
4410  return err;
4411  (*nb_bufs)++;
4412 
4413  for (int i = 0; i < planes; i++)
4414  region[i].bufferOffset = ((FFVkBuffer *)dst[0]->data)->virtual_offset +
4415  swf->data[i] - swf->data[0];
4416  } else if (nb_src_bufs == planes) { /* One buffer per plane */
4417  for (int i = 0; i < planes; i++) {
4418  err = ff_vk_host_map_buffer(&p->vkctx, &dst[i],
4419  swf->data[i], swf->buf[i],
4420  buf_usage);
4421  if (err < 0)
4422  goto fail;
4423  (*nb_bufs)++;
4424 
4425  region[i].bufferOffset = ((FFVkBuffer *)dst[i]->data)->virtual_offset;
4426  }
4427  } else {
4428  /* Weird layout (3 planes, 2 buffers), patch welcome, fallback to copy */
4429  return AVERROR_PATCHWELCOME;
4430  }
4431 
4432  return 0;
4433 
4434 fail:
4435  for (int i = 0; i < (*nb_bufs); i++)
4436  av_buffer_unref(&dst[i]);
4437  return err;
4438 }
4439 
4440 static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf,
4441  AVFrame *swf, int upload)
4442 {
4443  VulkanFramesPriv *fp = hwfc->hwctx;
4444  AVVulkanFramesContext *hwfc_vk = &fp->p;
4445  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4446  AVVulkanDeviceContext *hwctx = &p->p;
4447  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4448 
4449  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4450  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4451  const int planes = av_pix_fmt_count_planes(swf->format);
4452  const int nb_images = ff_vk_count_images(hwf_vk);
4453 
4454  VkSemaphoreWaitInfo sem_wait;
4455  VkHostImageLayoutTransitionInfoEXT layout_ch_info[AV_NUM_DATA_POINTERS];
4456  int nb_layout_ch = 0;
4457 
4458  hwfc_vk->lock_frame(hwfc, hwf_vk);
4459 
4460  for (int i = 0; i < nb_images; i++) {
4461  int compat = 0;
4462  for (int j = 0; j < p->vkctx.host_image_props.copySrcLayoutCount; j++) {
4463  if (hwf_vk->layout[i] == p->vkctx.host_image_props.pCopySrcLayouts[j]) {
4464  compat = 1;
4465  break;
4466  }
4467  }
4468  if (compat)
4469  continue;
4470 
4471  layout_ch_info[nb_layout_ch] = (VkHostImageLayoutTransitionInfoEXT) {
4472  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
4473  .image = hwf_vk->img[i],
4474  .oldLayout = hwf_vk->layout[i],
4475  .newLayout = VK_IMAGE_LAYOUT_GENERAL,
4476  .subresourceRange = {
4477  .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
4478  .levelCount = 1,
4479  .layerCount = 1,
4480  },
4481  };
4482 
4483  hwf_vk->layout[i] = layout_ch_info[nb_layout_ch].newLayout;
4484  nb_layout_ch++;
4485  }
4486 
4487  sem_wait = (VkSemaphoreWaitInfo) {
4488  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4489  .pSemaphores = hwf_vk->sem,
4490  .pValues = hwf_vk->sem_value,
4491  .semaphoreCount = nb_images,
4492  };
4493 
4494  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
4495 
4496  if (nb_layout_ch)
4497  vk->TransitionImageLayoutEXT(hwctx->act_dev,
4498  nb_layout_ch, layout_ch_info);
4499 
4500  if (upload) {
4501  VkMemoryToImageCopyEXT region_info = {
4502  .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT,
4503  .imageSubresource = {
4504  .layerCount = 1,
4505  },
4506  };
4507  VkCopyMemoryToImageInfoEXT copy_info = {
4508  .sType = VK_STRUCTURE_TYPE_COPY_MEMORY_TO_IMAGE_INFO_EXT,
4509  .regionCount = 1,
4510  .pRegions = &region_info,
4511  };
4512  for (int i = 0; i < planes; i++) {
4513  int img_idx = FFMIN(i, (nb_images - 1));
4514  uint32_t p_w, p_h;
4515  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4516 
4517  region_info.pHostPointer = swf->data[i];
4518  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4519  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4520  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4521  copy_info.dstImage = hwf_vk->img[img_idx];
4522  copy_info.dstImageLayout = hwf_vk->layout[img_idx];
4523 
4524  vk->CopyMemoryToImageEXT(hwctx->act_dev, &copy_info);
4525  }
4526  } else {
4527  VkImageToMemoryCopyEXT region_info = {
4528  .sType = VK_STRUCTURE_TYPE_IMAGE_TO_MEMORY_COPY_EXT,
4529  .imageSubresource = {
4530  .layerCount = 1,
4531  },
4532  };
4533  VkCopyImageToMemoryInfoEXT copy_info = {
4534  .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_MEMORY_INFO_EXT,
4535  .regionCount = 1,
4536  .pRegions = &region_info,
4537  };
4538  for (int i = 0; i < planes; i++) {
4539  int img_idx = FFMIN(i, (nb_images - 1));
4540  uint32_t p_w, p_h;
4541  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4542 
4543  region_info.pHostPointer = swf->data[i];
4544  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4545  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4546  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4547  copy_info.srcImage = hwf_vk->img[img_idx];
4548  copy_info.srcImageLayout = hwf_vk->layout[img_idx];
4549 
4550  vk->CopyImageToMemoryEXT(hwctx->act_dev, &copy_info);
4551  }
4552  }
4553 
4554  hwfc_vk->unlock_frame(hwfc, hwf_vk);
4555 
4556  return 0;
4557 }
4558 
4559 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4560  AVFrame *swf, AVFrame *hwf,
4561  int upload)
4562 {
4563  int err;
4564  VulkanFramesPriv *fp = hwfc->hwctx;
4565  AVVulkanFramesContext *hwctx = &fp->p;
4566  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4567  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4568 
4569  int host_mapped = 0;
4570 
4571  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4572  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4573 
4574  const int planes = av_pix_fmt_count_planes(swf->format);
4575  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4576  const int nb_images = ff_vk_count_images(hwf_vk);
4577 
4578  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4579  int nb_img_bar = 0;
4580 
4581  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4582  int nb_bufs = 0;
4583 
4584  VkCommandBuffer cmd_buf;
4585  FFVkExecContext *exec;
4586 
4587  /* Sanity checking */
4588  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4589  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4590  return AVERROR(EINVAL);
4591  }
4592 
4593  if (swf->width > hwfc->width || swf->height > hwfc->height)
4594  return AVERROR(EINVAL);
4595 
4596  if (hwctx->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
4597  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY))
4598  return vulkan_transfer_host(hwfc, hwf, swf, upload);
4599 
4600  for (int i = 0; i < av_pix_fmt_count_planes(swf->format); i++) {
4601  uint32_t p_w, p_h;
4602  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4603 
4604  /* Buffer region for this plane */
4605  region[i] = (VkBufferImageCopy) {
4606  .bufferOffset = 0,
4607  .bufferRowLength = swf->linesize[i],
4608  .bufferImageHeight = p_h,
4609  .imageSubresource.layerCount = 1,
4610  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4611  /* Rest of the fields adjusted/filled in later */
4612  };
4613  }
4614 
4615  /* Setup buffers first */
4616  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY && !p->avoid_host_import) {
4617  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4618  if (err >= 0)
4619  host_mapped = 1;
4620  }
4621 
4622  if (!host_mapped) {
4623  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4624  if (err < 0)
4625  goto end;
4626  nb_bufs = 1;
4627 
4628  if (upload) {
4629  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4630  if (err < 0)
4631  goto end;
4632  }
4633  }
4634 
4635  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4636  cmd_buf = exec->buf;
4637 
4638  ff_vk_exec_start(&p->vkctx, exec);
4639 
4640  /* Prep destination Vulkan frame */
4641  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4642  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4643  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4644  if (err < 0)
4645  goto end;
4646 
4647  /* No need to declare buf deps for synchronous transfers (downloads) */
4648  if (upload) {
4649  /* Add the software frame backing the buffers if we're host mapping */
4650  if (host_mapped) {
4651  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4652  if (err < 0) {
4653  ff_vk_exec_discard_deps(&p->vkctx, exec);
4654  goto end;
4655  }
4656  }
4657 
4658  /* Add the buffers as a dependency */
4659  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4660  if (err < 0) {
4661  ff_vk_exec_discard_deps(&p->vkctx, exec);
4662  goto end;
4663  }
4664  }
4665 
4666  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4667  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4668  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4669  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4670  VK_ACCESS_TRANSFER_READ_BIT,
4671  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4672  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4673  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
4674 
4675  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4676  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4677  .pImageMemoryBarriers = img_bar,
4678  .imageMemoryBarrierCount = nb_img_bar,
4679  });
4680 
4681  for (int i = 0; i < planes; i++) {
4682  int buf_idx = FFMIN(i, (nb_bufs - 1));
4683  int img_idx = FFMIN(i, (nb_images - 1));
4684  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4685 
4686  uint32_t orig_stride = region[i].bufferRowLength;
4687  region[i].bufferRowLength /= desc->comp[i].step;
4688  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4689 
4690  if (upload)
4691  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4692  hwf_vk->img[img_idx],
4693  img_bar[img_idx].newLayout,
4694  1, &region[i]);
4695  else
4696  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4697  img_bar[img_idx].newLayout,
4698  vkbuf->buf,
4699  1, &region[i]);
4700 
4701  region[i].bufferRowLength = orig_stride;
4702  }
4703 
4704  err = ff_vk_exec_submit(&p->vkctx, exec);
4705  if (err < 0) {
4706  ff_vk_exec_discard_deps(&p->vkctx, exec);
4707  } else if (!upload) {
4708  ff_vk_exec_wait(&p->vkctx, exec);
4709  if (!host_mapped)
4710  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4711  }
4712 
4713 end:
4714  for (int i = 0; i < nb_bufs; i++)
4715  av_buffer_unref(&bufs[i]);
4716 
4717  return err;
4718 }
4719 
4720 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4721  const AVFrame *src)
4722 {
4723  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4724 
4725  switch (src->format) {
4726 #if CONFIG_CUDA
4727  case AV_PIX_FMT_CUDA:
4728 #ifdef _WIN32
4729  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4730  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4731 #else
4732  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4733  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4734 #endif
4735  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4736 #endif
4737  default:
4738  if (src->hw_frames_ctx)
4739  return AVERROR(ENOSYS);
4740  else
4741  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4742  }
4743 }
4744 
4745 #if CONFIG_CUDA
4746 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4747  const AVFrame *src)
4748 {
4749  int err;
4750  CUcontext dummy;
4751  AVVkFrame *dst_f;
4752  AVVkFrameInternal *dst_int;
4753  VulkanFramesPriv *fp = hwfc->hwctx;
4754  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4755  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4756  int nb_images;
4757 
4758  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4759  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4760  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4761  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4762  CudaFunctions *cu = cu_internal->cuda_dl;
4763  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4764  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4765 
4766  dst_f = (AVVkFrame *)src->data[0];
4767  nb_images = ff_vk_count_images(dst_f);
4768 
4769  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4770  if (err < 0)
4771  return err;
4772 
4773  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4774  if (err < 0)
4775  return err;
4776 
4777  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4778  if (err < 0) {
4779  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4780  return err;
4781  }
4782 
4783  dst_int = dst_f->internal;
4784 
4785  for (int i = 0; i < planes; i++) {
4786  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4787  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4788  }
4789 
4790  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4791  nb_images, cuda_dev->stream));
4792  if (err < 0)
4793  goto fail;
4794 
4795  for (int i = 0; i < planes; i++) {
4796  CUDA_MEMCPY2D cpy = {
4797  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4798  .dstDevice = (CUdeviceptr)dst->data[i],
4799  .dstPitch = dst->linesize[i],
4800  .dstY = 0,
4801 
4802  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4803  .srcArray = dst_int->cu_array[i],
4804  };
4805 
4806  int w, h;
4807  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4808 
4809  cpy.WidthInBytes = w * desc->comp[i].step;
4810  cpy.Height = h;
4811 
4812  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4813  if (err < 0)
4814  goto fail;
4815  }
4816 
4817  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4818  nb_images, cuda_dev->stream));
4819  if (err < 0)
4820  goto fail;
4821 
4822  for (int i = 0; i < planes; i++)
4823  dst_f->sem_value[i]++;
4824 
4825  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4826 
4827  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4828 
4829  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4830 
4831 fail:
4832  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4833  vulkan_free_internal(dst_f);
4834  av_buffer_unref(&dst->buf[0]);
4835  return err;
4836 }
4837 #endif
4838 
4839 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4840  const AVFrame *src)
4841 {
4842  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4843 
4844  switch (dst->format) {
4845 #if CONFIG_CUDA
4846  case AV_PIX_FMT_CUDA:
4847 #ifdef _WIN32
4848  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4849  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4850 #else
4851  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4852  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4853 #endif
4854  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4855 #endif
4856  default:
4857  if (dst->hw_frames_ctx)
4858  return AVERROR(ENOSYS);
4859  else
4860  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4861  }
4862 }
4863 
4864 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4865  AVHWFramesContext *src_fc, int flags)
4866 {
4867  return vulkan_frames_init(dst_fc);
4868 }
4869 
4870 AVVkFrame *av_vk_frame_alloc(void)
4871 {
4872  int err;
4873  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4874  if (!f)
4875  return NULL;
4876 
4877  f->internal = av_mallocz(sizeof(*f->internal));
4878  if (!f->internal) {
4879  av_free(f);
4880  return NULL;
4881  }
4882 
4883  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4884  if (err != 0) {
4885  av_free(f->internal);
4886  av_free(f);
4887  return NULL;
4888  }
4889 
4890  return f;
4891 }
4892 
4893 const HWContextType ff_hwcontext_type_vulkan = {
4894  .type = AV_HWDEVICE_TYPE_VULKAN,
4895  .name = "Vulkan",
4896 
4897  .device_hwctx_size = sizeof(VulkanDevicePriv),
4898  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4899 
4900  .device_init = &vulkan_device_init,
4901  .device_uninit = &vulkan_device_uninit,
4902  .device_create = &vulkan_device_create,
4903  .device_derive = &vulkan_device_derive,
4904 
4905  .frames_get_constraints = &vulkan_frames_get_constraints,
4906  .frames_init = vulkan_frames_init,
4907  .frames_get_buffer = vulkan_get_buffer,
4908  .frames_uninit = vulkan_frames_uninit,
4909 
4910  .transfer_get_formats = vulkan_transfer_get_formats,
4911  .transfer_data_to = vulkan_transfer_data_to,
4912  .transfer_data_from = vulkan_transfer_data_from,
4913 
4914  .map_to = vulkan_map_to,
4915  .map_from = vulkan_map_from,
4916  .frames_derive_to = &vulkan_frames_derive_to,
4917 
4918  .pix_fmts = (const enum AVPixelFormat []) {
4919  AV_PIX_FMT_VULKAN,
4920  AV_PIX_FMT_NONE
4921  },
4922 };
flags
const SwsFlags flags[]
Definition: swscale.c:72
vulkan_loader.h
formats
formats
Definition: signature.h:47
AV_PIX_FMT_YUVA422P16
#define AV_PIX_FMT_YUVA422P16
Definition: pixfmt.h:596
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:629
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:88
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1906
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:565
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:147
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:216
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:129
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:668
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:401
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4383
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_PIX_FMT_GRAY32
#define AV_PIX_FMT_GRAY32
Definition: pixfmt.h:523
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:214
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
HWMapDescriptor::source
AVFrame * source
A reference to the original source of the mapping.
Definition: hwcontext_internal.h:124
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4720
FF_VK_EXT_EXTERNAL_WIN32_MEMORY
#define FF_VK_EXT_EXTERNAL_WIN32_MEMORY
Definition: vulkan_functions.h:39
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:58
thread.h
vk_dbg_callback
static VKAPI_ATTR VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:767
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3456
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:357
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:73
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:519
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2268
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:60
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:196
av_unused
#define av_unused
Definition: attributes.h:156
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4864
VulkanDeviceFeatures::explicit_mem_layout
VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout
Definition: hwcontext_vulkan.c:84
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:64
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:64
mode
Definition: swscale.c:60
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:427
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:679
AVFrame::width
int width
Definition: frame.h:499
AV_PIX_FMT_YUVA420P16
#define AV_PIX_FMT_YUVA420P16
Definition: pixfmt.h:595
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:608
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2702
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AV_PIX_FMT_YUVA420P10
#define AV_PIX_FMT_YUVA420P10
Definition: pixfmt.h:590
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:251
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:375
pthread_mutex_lock
static av_always_inline int pthread_mutex_lock(pthread_mutex_t *mutex)
Definition: os2threads.h:119
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:793
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:513
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:156
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:630
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:539
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
ff_vk_flush_buffer
int ff_vk_flush_buffer(FFVulkanContext *s, FFVkBuffer *buf, VkDeviceSize offset, VkDeviceSize mem_size, int flush)
Flush or invalidate a single buffer, with a given size and offset.
Definition: vulkan.c:1171
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:296
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:226
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
switch_new_props
static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout, VkAccessFlags2 *new_access)
Definition: hwcontext_vulkan.c:2526
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:847
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4275
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:397
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:548
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2836
AVDictionary
Definition: dict.c:32
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:741
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:44
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:449
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:220
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:604
AV_PIX_FMT_YUVA422P10
#define AV_PIX_FMT_YUVA422P10
Definition: pixfmt.h:591
dummy
static int dummy
Definition: ffplay.c:3751
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1302
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:780
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:845
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:618
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:448
AVVulkanDeviceContext::unlock_queue
attribute_deprecated void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:189
av_vk_get_optional_device_extensions
const char ** av_vk_get_optional_device_extensions(int *count)
Returns an array of optional Vulkan device extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:752
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:126
VulkanDeviceFeatures::host_image_copy
VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy
Definition: hwcontext_vulkan.c:83
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2314
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
VulkanDevicePriv::compute_qf
AVVulkanDeviceQueueFamily * compute_qf
Definition: hwcontext_vulkan.c:132
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:444
FF_VK_EXT_HOST_IMAGE_COPY
#define FF_VK_EXT_HOST_IMAGE_COPY
Definition: vulkan_functions.h:51
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
FF_VK_EXT_EXPECT_ASSUME
#define FF_VK_EXT_EXPECT_ASSUME
Definition: vulkan_functions.h:49
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2522
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:35
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:102
close
static av_cold void close(AVCodecParserContext *s)
Definition: apv_parser.c:197
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3496
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:394
FF_VK_EXT_LONG_VECTOR
#define FF_VK_EXT_LONG_VECTOR
Definition: vulkan_functions.h:55
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:138
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2165
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:669
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:560
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:315
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1305
AVVulkanDeviceContext::lock_queue
attribute_deprecated void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:181
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2949
fail
#define fail()
Definition: checkasm.h:223
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:714
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:558
AV_PIX_FMT_YUVA444P16
#define AV_PIX_FMT_YUVA444P16
Definition: pixfmt.h:597
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:267
VulkanDevicePriv
Definition: hwcontext_vulkan.c:122
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:139
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:328
switch_layout
static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2561
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:288
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:220
ff_vk_frame_barrier
void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, AVFrame *pic, VkImageMemoryBarrier2 *bar, int *nb_bar, VkPipelineStageFlags2 src_stage, VkPipelineStageFlags2 dst_stage, VkAccessFlagBits2 new_access, VkImageLayout new_layout, uint32_t new_qf)
Definition: vulkan.c:2050
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:462
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1894
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::limit_queues
int limit_queues
Definition: hwcontext_vulkan.c:169
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:131
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:611
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:70
ff_vk_host_map_buffer
int ff_vk_host_map_buffer(FFVulkanContext *s, AVBufferRef **dst, uint8_t *src_data, const AVBufferRef *src_buf, VkBufferUsageFlags usage)
Maps a system RAM buffer into a Vulkan buffer.
Definition: vulkan.c:1391
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:40
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:522
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
FF_VK_EXT_EXTERNAL_WIN32_SEM
#define FF_VK_EXT_EXTERNAL_WIN32_SEM
Definition: vulkan_functions.h:40
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:166
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:136
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1532
VulkanFramesPriv::drm_format_modifier_properties
VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5]
Definition: hwcontext_vulkan.c:192
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:523
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:63
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:52
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:542
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:606
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:210
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
ff_vk_link_struct
static void ff_vk_link_struct(void *chain, const void *in)
Definition: vulkan.h:388
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1017
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:551
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:158
AVHWFramesContext::height
int height
Definition: hwcontext.h:220
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:456
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4066
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:60
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:562
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:181
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1308
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:564
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:563
AV_PIX_FMT_YUVA420P
@ AV_PIX_FMT_YUVA420P
planar YUV 4:2:0, 20bpp, (1 Cr & Cb sample per 2x2 Y & A samples)
Definition: pixfmt.h:108
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:629
pthread_mutex_unlock
static av_always_inline int pthread_mutex_unlock(pthread_mutex_t *mutex)
Definition: os2threads.h:126
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:552
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:128
VulkanDevicePriv::ext_sem_props_opaque
VkExternalSemaphoreProperties ext_sem_props_opaque
Definition: hwcontext_vulkan.c:142
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2669
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4559
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:42
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:553
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:179
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:393
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1774
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:494
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:301
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:42
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:273
AV_PIX_FMT_GBRAP32
#define AV_PIX_FMT_GBRAP32
Definition: pixfmt.h:566
AV_PIX_FMT_YUVA444P12
#define AV_PIX_FMT_YUVA444P12
Definition: pixfmt.h:594
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2959
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:231
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:550
ctx
static AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:521
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:620
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1310
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:626
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:176
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:3215
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
VulkanDevicePriv::avoid_host_import
int avoid_host_import
Definition: hwcontext_vulkan.c:166
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:299
av_mallocz
#define av_mallocz(s)
Definition: tableprint_vlc.h:31
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:840
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:582
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
tmp
static uint8_t tmp[40]
Definition: aes_ctr.c:52
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:402
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2442
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:519
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:515
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:148
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:561
opts
static AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2518
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:529
planes
static const struct @578 planes[]
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:213
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
format
New swscale design to change SwsGraph is what coordinates multiple passes These can include cascaded scaling error diffusion and so on Or we could have separate passes for the vertical and horizontal scaling In between each SwsPass lies a fully allocated image buffer Graph passes may have different levels of e g we can have a single threaded error diffusion pass following a multi threaded scaling pass SwsGraph is internally recreated whenever the image format
Definition: swscale-v2.txt:14
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:33
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:117
FFVkFormatEntry
Definition: hwcontext_vulkan.c:396
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:46
AV_DRM_MAX_PLANES
@ AV_DRM_MAX_PLANES
The maximum number of layers/planes in a DRM frame.
Definition: hwcontext_drm.h:39
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:195
FF_VK_EXT_VIDEO_DECODE_VP9
#define FF_VK_EXT_VIDEO_DECODE_VP9
Definition: vulkan_functions.h:65
FF_VK_EXT_SUBGROUP_ROTATE
#define FF_VK_EXT_SUBGROUP_ROTATE
Definition: vulkan_functions.h:50
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:68
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:154
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:841
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:260
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1554
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:168
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:617
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:201
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4893
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:137
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:103
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:281
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:240
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:540
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2447
FFVkBuffer::mapped_mem
uint8_t * mapped_mem
Definition: vulkan.h:134
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:200
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:312
exp
int8_t exp
Definition: eval.c:76
FF_VK_EXT_REPLICATED_COMPOSITES
#define FF_VK_EXT_REPLICATED_COMPOSITES
Definition: vulkan_functions.h:54
AV_PIX_FMT_GBRPF16
#define AV_PIX_FMT_GBRPF16
Definition: pixfmt.h:576
VulkanFramesPriv
Definition: hwcontext_vulkan.c:172
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2408
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagsKHR flags)
Definition: hwcontext_vulkan.c:1525
index
int index
Definition: gxfenc.c:90
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:531
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1301
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:256
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:149
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:62
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2521
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:329
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2874
VulkanDevicePriv::dprops
VkPhysicalDeviceDriverProperties dprops
Definition: hwcontext_vulkan.c:139
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2516
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:59
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1306
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:603
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:87
AVVkFrame
Definition: hwcontext_vulkan.h:310
i
#define i(width, name, range_min, range_max)
Definition: cbs_h264.c:63
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4870
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:843
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:32
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1751
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:424
FF_VK_EXT_NO_FLAG
#define FF_VK_EXT_NO_FLAG
Definition: vulkan_functions.h:74
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:578
AV_PIX_FMT_YUV422P12
#define AV_PIX_FMT_YUV422P12
Definition: pixfmt.h:544
VulkanDeviceFeatures::cooperative_matrix
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix
Definition: hwcontext_vulkan.c:114
AV_PIX_FMT_RGB48
#define AV_PIX_FMT_RGB48
Definition: pixfmt.h:525
size
int size
Definition: twinvq_data.h:10344
ff_vk_exec_add_dep_sw_frame
int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f)
Definition: vulkan.c:647
vulkan_transfer_data_from
static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4839
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:428
FF_VK_EXT_PUSH_DESCRIPTOR
#define FF_VK_EXT_PUSH_DESCRIPTOR
Definition: vulkan_functions.h:47
VulkanDevicePriv::use_linear_images
int use_linear_images
Definition: hwcontext_vulkan.c:157
AV_PIX_FMT_YUV444P12
#define AV_PIX_FMT_YUV444P12
Definition: pixfmt.h:546
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:514
AV_PIX_FMT_NV16
@ AV_PIX_FMT_NV16
interleaved chroma YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:198
VulkanDeviceSelection::vendor_id
uint32_t vendor_id
Definition: hwcontext_vulkan.c:1309
PREP_MODE_GENERAL
@ PREP_MODE_GENERAL
Definition: hwcontext_vulkan.c:2517
AV_PIX_FMT_Y212
#define AV_PIX_FMT_Y212
Definition: pixfmt.h:607
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:128
AVDRMObjectDescriptor::size
size_t size
Total size of the object.
Definition: hwcontext_drm.h:58
AV_PIX_FMT_YUVA444P
@ AV_PIX_FMT_YUVA444P
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:174
VulkanFramesPriv::upload_exec
FFVkExecPool upload_exec
Definition: hwcontext_vulkan.c:182
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AV_PIX_FMT_YUVA444P10
#define AV_PIX_FMT_YUVA444P10
Definition: pixfmt.h:592
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
AVHWFramesConstraints::max_width
int max_width
The maximum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:469
FFVkExecContext
Definition: vulkan.h:145
VulkanOptExtension
Definition: hwcontext_vulkan.c:667
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
AV_PIX_FMT_P216
#define AV_PIX_FMT_P216
Definition: pixfmt.h:620
CHECK_CU
#define CHECK_CU(x)
Definition: cuviddec.c:117
AV_PIX_FMT_P210
#define AV_PIX_FMT_P210
Definition: pixfmt.h:616
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
VulkanDeviceFeatures::subgroup_rotate
VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate
Definition: hwcontext_vulkan.c:82
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:62
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:221
vulkan_free_internal
static void vulkan_free_internal(AVVkFrame *f)
Definition: hwcontext_vulkan.c:2372
VulkanDeviceFeatures::timeline_semaphore
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore
Definition: hwcontext_vulkan.c:81
AV_HWDEVICE_TYPE_VAAPI
@ AV_HWDEVICE_TYPE_VAAPI
Definition: hwcontext.h:31
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
FF_VK_EXT_EXTERNAL_HOST_MEMORY
#define FF_VK_EXT_EXTERNAL_HOST_MEMORY
Definition: vulkan_functions.h:36
FF_VK_EXT_EXPLICIT_MEM_LAYOUT
#define FF_VK_EXT_EXPLICIT_MEM_LAYOUT
Definition: vulkan_functions.h:53
AVDRMFrameDescriptor::objects
AVDRMObjectDescriptor objects[AV_DRM_MAX_PLANES]
Array of objects making up the frame.
Definition: hwcontext_drm.h:141
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
av_assert2
#define av_assert2(cond)
assert() equivalent, that does lie in speed critical code.
Definition: avassert.h:68
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:560
FF_VK_EXT_RELAXED_EXTENDED_INSTR
#define FF_VK_EXT_RELAXED_EXTENDED_INSTR
Definition: vulkan_functions.h:48
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:63
VulkanFramesPriv::compute_exec
FFVkExecPool compute_exec
Definition: hwcontext_vulkan.c:179
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:146
AVDRMObjectDescriptor::format_modifier
uint64_t format_modifier
Format modifier applied to the object (DRM_FORMAT_MOD_*).
Definition: hwcontext_drm.h:65
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
AV_PIX_FMT_GBRP12
#define AV_PIX_FMT_GBRP12
Definition: pixfmt.h:559
weights
static const int weights[]
Definition: hevc_pel.c:32
AV_PIX_FMT_NV24
@ AV_PIX_FMT_NV24
planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:371
plane_info
Definition: vf_edgedetect.c:53
VulkanDevicePriv::nb_img_qfs
uint32_t nb_img_qfs
Definition: hwcontext_vulkan.c:151
vulkan_frames_init
static int vulkan_frames_init(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2977
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
VulkanDeviceSelection::has_uuid
int has_uuid
Definition: hwcontext_vulkan.c:1303
hwcontext_drm.h
AVDRMPlaneDescriptor::object_index
int object_index
Index of the object containing this plane in the objects array of the enclosing frame descriptor.
Definition: hwcontext_drm.h:79
ff_hwframe_map_replace
int ff_hwframe_map_replace(AVFrame *dst, const AVFrame *src)
Replace the current hwmap of dst with the one from src, used for indirect mappings like VAAPI->(DRM)-...
Definition: hwcontext.c:948
VulkanDevicePriv::img_qfs
uint32_t img_qfs[64]
Definition: hwcontext_vulkan.c:150
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
mod
static int mod(int a, int b)
Modulo operation with only positive remainders.
Definition: vf_v360.c:755
AV_PIX_FMT_P016
#define AV_PIX_FMT_P016
Definition: pixfmt.h:604
AVHWFrameTransferDirection
AVHWFrameTransferDirection
Definition: hwcontext.h:406
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:348
create_instance
static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1164
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:118
FF_VK_EXT_EXTERNAL_FD_MEMORY
#define FF_VK_EXT_EXTERNAL_FD_MEMORY
Definition: vulkan_functions.h:34
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
hwcontext_vaapi.h
AVDRMLayerDescriptor::format
uint32_t format
Format of the layer (DRM_FORMAT_*).
Definition: hwcontext_drm.h:100
VulkanDevicePriv::transfer_qf
AVVulkanDeviceQueueFamily * transfer_qf
Definition: hwcontext_vulkan.c:133
ret
ret
Definition: filter_design.txt:187
pixfmt
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:367
AVHWDeviceContext::type
enum AVHWDeviceType type
This field identifies the underlying API used for hardware access.
Definition: hwcontext.h:75
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:96
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:265
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:137
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:316
cuda_check.h
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:153
VulkanFramesPriv::tmp
AVBufferPool * tmp
Definition: hwcontext_vulkan.c:186
vulkan_get_buffer
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
Definition: hwcontext_vulkan.c:3201
pick_queue_family
static int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf, VkQueueFlagBits flags)
Definition: hwcontext_vulkan.c:1496
FFVkExecPool
Definition: vulkan.h:290
vulkan_transfer_host
static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf, AVFrame *swf, int upload)
Definition: hwcontext_vulkan.c:4440
unlock_queue
static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1900
AVHWFramesConstraints::max_height
int max_height
Definition: hwcontext.h:470
AVVkFrame::internal
struct AVVkFrameInternal * internal
Internal data.
Definition: hwcontext_vulkan.h:361
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:286
AV_PIX_FMT_YUV420P12
#define AV_PIX_FMT_YUV420P12
Definition: pixfmt.h:543
AV_PIX_FMT_UYVY422
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:88
check_extensions
static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode debug_mode)
Definition: hwcontext_vulkan.c:852
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:156
vulkan_device_create
static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2149
AVFrame::height
int height
Definition: frame.h:499
PREP_MODE_ENCODING_DPB
@ PREP_MODE_ENCODING_DPB
Definition: hwcontext_vulkan.c:2523
FFVkFormatEntry::pixfmt
enum AVPixelFormat pixfmt
Definition: hwcontext_vulkan.c:398
AVHWFramesConstraints::min_height
int min_height
Definition: hwcontext.h:463
RELEASE_PROPS
#define RELEASE_PROPS(props, count)
Definition: hwcontext_vulkan.c:813
mode
mode
Definition: ebur128.h:83
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_PIX_FMT_YUVA422P12
#define AV_PIX_FMT_YUVA422P12
Definition: pixfmt.h:593
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:579
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:515
VulkanDevicePriv::feats
VulkanDeviceFeatures feats
Definition: hwcontext_vulkan.c:145
switch_layout_host
static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2624
LIBAVUTIL_VERSION_MICRO
#define LIBAVUTIL_VERSION_MICRO
Definition: version.h:83
AV_PIX_FMT_GBRAPF16
#define AV_PIX_FMT_GBRAPF16
Definition: pixfmt.h:577
find_device
static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
Definition: hwcontext_vulkan.c:1325
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:130
FF_VK_STRUCT_EXT
#define FF_VK_STRUCT_EXT(CTX, BASE, STRUCT_P, EXT_FLAG, TYPE)
Definition: vulkan.h:397
optional_instance_exts
static const VulkanOptExtension optional_instance_exts[]
Definition: hwcontext_vulkan.c:672
AV_PIX_FMT_FLAG_PLANAR
#define AV_PIX_FMT_FLAG_PLANAR
At least one pixel component is not in the first data plane.
Definition: pixdesc.h:132
ff_vk_map_feats_to_usage
VkImageUsageFlags ff_vk_map_feats_to_usage(VkFormatFeatureFlagBits2 feats)
Map between usage and features.
FF_VK_EXT_ATOMIC_FLOAT
#define FF_VK_EXT_ATOMIC_FLOAT
Definition: vulkan_functions.h:43
FFVkFormatEntry::fallback
const VkFormat fallback[5]
Definition: hwcontext_vulkan.c:403
Windows::Graphics::DirectX::Direct3D11::p
IDirect3DDxgiInterfaceAccess _COM_Outptr_ void ** p
Definition: vsrc_gfxcapture_winrt.hpp:53
PREP_MODE_EXTERNAL_IMPORT
@ PREP_MODE_EXTERNAL_IMPORT
Definition: hwcontext_vulkan.c:2520
AV_PIX_FMT_RGBAF32
#define AV_PIX_FMT_RGBAF32
Definition: pixfmt.h:627
FF_VK_EXT_VIDEO_DECODE_AV1
#define FF_VK_EXT_VIDEO_DECODE_AV1
Definition: vulkan_functions.h:66
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:229
VulkanDeviceSelection::drm_major
uint32_t drm_major
Definition: hwcontext_vulkan.c:1304
get_plane_buf
static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4344
AV_PIX_FMT_P010
#define AV_PIX_FMT_P010
Definition: pixfmt.h:602
unlock_frame
static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2954
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:356
AV_PIX_FMT_GBRP
@ AV_PIX_FMT_GBRP
planar GBR 4:4:4 24bpp
Definition: pixfmt.h:165
desc
const char * desc
Definition: libsvtav1.c:82
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:116
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:286
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
VulkanFramesPriv::download_exec
FFVkExecPool download_exec
Definition: hwcontext_vulkan.c:183
mem.h
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:340
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
av_strdup
#define av_strdup(s)
Definition: ops_asmgen.c:47
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
w
uint8_t w
Definition: llvidencdsp.c:39
hwcontext_internal.h
FF_VK_EXT_VIDEO_ENCODE_H264
#define FF_VK_EXT_VIDEO_ENCODE_H264
Definition: vulkan_functions.h:69
ADD_QUEUE
#define ADD_QUEUE(ctx_qf, qc, flag)
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:104
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
AVDictionaryEntry
Definition: dict.h:90
get_plane_wh
static void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format, int frame_w, int frame_h, int plane)
Definition: hwcontext_vulkan.c:2685
ff_vk_count_images
static int ff_vk_count_images(AVVkFrame *f)
Definition: vulkan.h:366
ff_vk_exec_discard_deps
void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:592
PREP_MODE_EXTERNAL_EXPORT
@ PREP_MODE_EXTERNAL_EXPORT
Definition: hwcontext_vulkan.c:2519
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
AV_PIX_FMT_P416
#define AV_PIX_FMT_P416
Definition: pixfmt.h:621
VulkanDevicePriv::mprops
VkPhysicalDeviceMemoryProperties mprops
Definition: hwcontext_vulkan.c:137
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FF_VULKAN_DEBUG_NB
@ FF_VULKAN_DEBUG_NB
Definition: hwcontext_vulkan.c:849
FFVkBuffer
Definition: vulkan.h:125
vk_dev_type
static const char * vk_dev_type(enum VkPhysicalDeviceType type)
Definition: hwcontext_vulkan.c:1313
VulkanDevicePriv::disable_multiplane
int disable_multiplane
Definition: hwcontext_vulkan.c:163
imgutils.h
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:905
AV_PIX_FMT_XV36
#define AV_PIX_FMT_XV36
Definition: pixfmt.h:610
hwcontext.h
AVDRMPlaneDescriptor::pitch
ptrdiff_t pitch
Pitch (linesize) of this plane.
Definition: hwcontext_drm.h:87
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:472
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFVkFormatEntry::vk_planes
int vk_planes
Definition: hwcontext_vulkan.c:400
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
HWContextType
Definition: hwcontext_internal.h:29
FF_VK_EXT_VIDEO_ENCODE_AV1
#define FF_VK_EXT_VIDEO_ENCODE_AV1
Definition: vulkan_functions.h:71
AV_PIX_FMT_P412
#define AV_PIX_FMT_P412
Definition: pixfmt.h:619
device_features_init
static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
Definition: hwcontext_vulkan.c:214
VulkanDevicePriv::contiguous_planes
int contiguous_planes
Definition: hwcontext_vulkan.c:160
AVVAAPIDeviceContext
VAAPI connection details.
Definition: hwcontext_vaapi.h:68
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVDictionaryEntry::value
char * value
Definition: dict.h:92
avstring.h
AVDRMDeviceContext
DRM device.
Definition: hwcontext_drm.h:157
AV_PIX_FMT_GRAY12
#define AV_PIX_FMT_GRAY12
Definition: pixfmt.h:520
copy_buffer_data
static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, AVFrame *swf, VkBufferImageCopy *region, int planes, int upload)
Definition: hwcontext_vulkan.c:4301
VulkanDeviceFeatures::atomic_float
VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float
Definition: hwcontext_vulkan.c:115
ADD_VAL_TO_LIST
#define ADD_VAL_TO_LIST(list, count, val)
Definition: hwcontext_vulkan.c:799
AV_PIX_FMT_BAYER_RGGB16
#define AV_PIX_FMT_BAYER_RGGB16
Definition: pixfmt.h:572
AVDRMFrameDescriptor::nb_objects
int nb_objects
Number of DRM objects making up this frame.
Definition: hwcontext_drm.h:137
VulkanDeviceFeatures::shader_object
VkPhysicalDeviceShaderObjectFeaturesEXT shader_object
Definition: hwcontext_vulkan.c:113
HWMapDescriptor
Definition: hwcontext_internal.h:120
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
FF_VK_EXT_ZERO_INITIALIZE
#define FF_VK_EXT_ZERO_INITIALIZE
Definition: vulkan_functions.h:52
FFVulkanFunctions
Definition: vulkan_functions.h:274
VulkanDevicePriv::p
AVVulkanDeviceContext p
The public AVVulkanDeviceContext.
Definition: hwcontext_vulkan.c:126
VulkanFramesPriv::modifier_info
VkImageDrmFormatModifierListCreateInfoEXT * modifier_info
Definition: hwcontext_vulkan.c:189
FFVkFormatEntry::aspect
VkImageAspectFlags aspect
Definition: hwcontext_vulkan.c:399
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1286
VulkanDeviceSelection::name
const char * name
Definition: hwcontext_vulkan.c:1307
src
#define src
Definition: vp8dsp.c:248
AV_HWDEVICE_TYPE_DRM
@ AV_HWDEVICE_TYPE_DRM
Definition: hwcontext.h:36
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:148
AV_PIX_FMT_YUVA422P
@ AV_PIX_FMT_YUVA422P
planar YUV 4:2:2 24bpp, (1 Cr & Cb sample per 2x1 Y & A samples)
Definition: pixfmt.h:173
vulkan_device_create_internal
static int vulkan_device_create_internal(AVHWDeviceContext *ctx, VulkanDeviceSelection *dev_select, int disable_multiplane, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1787
w32dlfcn.h
av_vk_get_optional_instance_extensions
const char ** av_vk_get_optional_instance_extensions(int *count)
Returns an array of optional Vulkan instance extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:738
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3376
vulkan_device_has_rebar
static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:820
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlags *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2812
Generated on Sat Apr 4 2026 19:23:44 for FFmpeg by   doxygen 1.8.17