FFmpeg
vulkan_decode.h
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifndef AVCODEC_VULKAN_DECODE_H
20 #define AVCODEC_VULKAN_DECODE_H
21 
22 #include "codec_id.h"
23 #include "decode.h"
24 #include "hwaccel_internal.h"
25 #include "internal.h"
26 
27 #include "vulkan_video.h"
28 
29 typedef struct FFVulkanDecodeDescriptor {
32  VkVideoCodecOperationFlagBitsKHR decode_op;
33 
34  VkExtensionProperties ext_props;
36 
37 typedef struct FFVulkanDecodeProfileData {
38  VkVideoDecodeH264ProfileInfoKHR h264_profile;
39  VkVideoDecodeH265ProfileInfoKHR h265_profile;
40  VkVideoDecodeAV1ProfileInfoKHR av1_profile;
41  VkVideoDecodeUsageInfoKHR usage;
42  VkVideoProfileInfoKHR profile;
43  VkVideoProfileListInfoKHR profile_list;
45 
46 typedef struct FFVulkanDecodeShared {
48  FFVkVideoCommon common;
50 
51  VkVideoCapabilitiesKHR caps;
52  VkVideoDecodeCapabilitiesKHR dec_caps;
53 
54  AVBufferRef *dpb_hwfc_ref; /* Only used for dedicated_dpb */
55 
56  AVFrame *layered_frame; /* Only used for layered_dpb */
57  VkImageView layered_view;
58  VkImageAspectFlags layered_aspect;
59 
60  VkVideoSessionParametersKHR empty_session_params;
61 
62  VkSamplerYcbcrConversion yuv_sampler;
64 
65 typedef struct FFVulkanDecodeContext {
69 
70  int dedicated_dpb; /* Oddity #1 - separate DPB images */
71  int layered_dpb; /* Madness #1 - layered DPB images */
72  int external_fg; /* Oddity #2 - hardware can't apply film grain */
73  uint32_t frame_id_alloc_mask; /* For AV1 only */
74 
75  /* Workaround for NVIDIA drivers tested with CTS version 1.3.8 for AV1.
76  * The tests were incorrect as the OrderHints were offset by 1. */
78 
79  /* Thread-local state below */
82 
83  uint32_t *slice_off;
84  unsigned int slice_off_max;
86 
87 typedef struct FFVulkanDecodePicture {
88  AVFrame *dpb_frame; /* Only used for out-of-place decoding. */
89 
90  VkImageView img_view_ref; /* Image representation view (reference) */
91  VkImageView img_view_out; /* Image representation view (output-only) */
92  VkImageView img_view_dest; /* Set to img_view_out if no layered refs are used */
93  VkImageAspectFlags img_aspect; /* Image plane mask bits */
94  VkImageAspectFlags img_aspect_ref; /* Only used for out-of-place decoding */
95 
96  VkSemaphore sem;
97  uint64_t sem_value;
98 
99  /* Current picture */
100  VkVideoPictureResourceInfoKHR ref;
101  VkVideoReferenceSlotInfoKHR ref_slot;
102 
103  /* Picture refs. H264 has the maximum number of refs (36) of any supported codec. */
104  VkVideoPictureResourceInfoKHR refs [36];
105  VkVideoReferenceSlotInfoKHR ref_slots[36];
106 
107  /* Main decoding struct */
108  VkVideoDecodeInfoKHR decode_info;
109 
110  /* Slice data */
112  size_t slices_size;
113 
114  /* Vulkan functions needed for destruction, as no other context is guaranteed to exist */
115  PFN_vkWaitSemaphores wait_semaphores;
116  PFN_vkDestroyImageView destroy_image_view;
118 
119 /**
120  * Initialize decoder.
121  */
123 
124 /**
125  * Synchronize the contexts between 2 threads.
126  */
128 
129 /**
130  * Initialize hw_frames_ctx with the parameters needed to decode the stream
131  * using the parameters from avctx.
132  *
133  * NOTE: if avctx->internal->hwaccel_priv_data exists, will partially initialize
134  * the context.
135  */
136 int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx);
137 
138 /**
139  * Removes current session parameters to recreate them
140  */
141 int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s);
142 
143 /**
144  * Prepare a frame, creates the image view, and sets up the dpb fields.
145  */
147  FFVulkanDecodePicture *vkpic, int is_current,
148  int alloc_dpb);
149 
150 /**
151  * Add slice data to frame.
152  */
154  const uint8_t *data, size_t size, int add_startcode,
155  uint32_t *nb_slices, const uint32_t **offsets);
156 
157 /**
158  * Decode a frame.
159  */
161  AVFrame *pic, FFVulkanDecodePicture *vp,
162  AVFrame *rpic[], FFVulkanDecodePicture *rvkp[]);
163 
164 /**
165  * Free a frame and its state.
166  */
168 
169 /**
170  * Get an FFVkBuffer suitable for decoding from.
171  */
173  void *create_pNext, size_t size);
174 
175 /**
176  * Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
177  */
178 int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx,
179  const VkVideoSessionParametersCreateInfoKHR *session_params_create);
180 
181 /**
182  * Flush decoder.
183  */
185 
186 /**
187  * Free decoder.
188  */
190 
191 #endif /* AVCODEC_VULKAN_DECODE_H */
FFVulkanDecodeShared::layered_view
VkImageView layered_view
Definition: vulkan_decode.h:57
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:112
FFVulkanDecodeShared::s
FFVulkanContext s
Definition: vulkan_decode.h:47
FFVulkanDecodeProfileData::profile
VkVideoProfileInfoKHR profile
Definition: vulkan_decode.h:42
FFVulkanDecodeProfileData::h265_profile
VkVideoDecodeH265ProfileInfoKHR h265_profile
Definition: vulkan_decode.h:39
FFVulkanDecodeContext::shared_ctx
FFVulkanDecodeShared * shared_ctx
Definition: vulkan_decode.h:66
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1092
FFVulkanDecodeContext::exec_pool
FFVkExecPool exec_pool
Definition: vulkan_decode.h:68
FFVulkanDecodeContext::hevc_headers_size
size_t hevc_headers_size
Definition: vulkan_decode.h:81
FFVulkanDecodeShared::common
FFVkVideoCommon common
Definition: vulkan_decode.h:48
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:344
internal.h
FFVulkanDecodeShared::layered_frame
AVFrame * layered_frame
Definition: vulkan_decode.h:56
b
#define b
Definition: input.c:41
data
const char data[16]
Definition: mxf.c:148
FFVulkanDecodeDescriptor::codec_id
enum AVCodecID codec_id
Definition: vulkan_decode.h:30
FFVulkanDecodeContext::frame_id_alloc_mask
uint32_t frame_id_alloc_mask
Definition: vulkan_decode.h:73
FFVulkanDecodeShared::yuv_sampler
VkSamplerYcbcrConversion yuv_sampler
Definition: vulkan_decode.h:62
ff_vk_get_decode_buffer
int ff_vk_get_decode_buffer(FFVulkanDecodeContext *ctx, AVBufferRef **buf, void *create_pNext, size_t size)
Get an FFVkBuffer suitable for decoding from.
FFVulkanDecodeProfileData::av1_profile
VkVideoDecodeAV1ProfileInfoKHR av1_profile
Definition: vulkan_decode.h:40
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:250
FFVulkanDecodeContext
Definition: vulkan_decode.h:65
FFVulkanDecodeShared::layered_aspect
VkImageAspectFlags layered_aspect
Definition: vulkan_decode.h:58
FFVulkanDecodePicture::ref
VkVideoPictureResourceInfoKHR ref
Definition: vulkan_decode.h:100
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:67
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1063
FFVulkanDecodeDescriptor::decode_op
VkVideoCodecOperationFlagBitsKHR decode_op
Definition: vulkan_decode.h:32
FFVulkanDecodePicture::sem_value
uint64_t sem_value
Definition: vulkan_decode.h:97
FFVulkanDecodeShared
Definition: vulkan_decode.h:46
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:60
FFVulkanDecodeContext::slice_off
uint32_t * slice_off
Definition: vulkan_decode.h:83
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:91
FFVulkanDecodeProfileData::h264_profile
VkVideoDecodeH264ProfileInfoKHR h264_profile
Definition: vulkan_decode.h:38
codec_id.h
FFVulkanDecodePicture::refs
VkVideoPictureResourceInfoKHR refs[36]
Definition: vulkan_decode.h:104
FFVulkanDecodeProfileData::profile_list
VkVideoProfileListInfoKHR profile_list
Definition: vulkan_decode.h:43
FFVulkanDecodePicture::wait_semaphores
PFN_vkWaitSemaphores wait_semaphores
Definition: vulkan_decode.h:115
s
#define s(width, name)
Definition: cbs_vp9.c:198
FFVulkanDecodePicture
Definition: vulkan_decode.h:87
offsets
static const int offsets[]
Definition: hevc_pel.c:34
ctx
AVFormatContext * ctx
Definition: movenc.c:48
decode.h
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:120
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:79
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:90
hwaccel_internal.h
FFVulkanDecodeProfileData::usage
VkVideoDecodeUsageInfoKHR usage
Definition: vulkan_decode.h:41
FFVulkanDecodeDescriptor::decode_extension
FFVulkanExtensions decode_extension
Definition: vulkan_decode.h:31
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:572
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:342
FFVulkanContext
Definition: vulkan.h:228
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
FFVulkanDecodeShared::dpb_hwfc_ref
AVBufferRef * dpb_hwfc_ref
Definition: vulkan_decode.h:54
FFVulkanDecodeShared::caps
VkVideoCapabilitiesKHR caps
Definition: vulkan_decode.h:51
size
int size
Definition: twinvq_data.h:10344
FFVkQueueFamilyCtx
Definition: vulkan.h:110
FFVulkanDecodePicture::img_aspect_ref
VkImageAspectFlags img_aspect_ref
Definition: vulkan_decode.h:94
FFVulkanDecodeShared::empty_session_params
VkVideoSessionParametersKHR empty_session_params
Definition: vulkan_decode.h:60
FFVulkanDecodeContext::slice_off_max
unsigned int slice_off_max
Definition: vulkan_decode.h:84
HEVCHeaderSet
Definition: vulkan_hevc.c:65
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1006
FFVulkanDecodeProfileData
Definition: vulkan_decode.h:37
FFVulkanDecodeDescriptor::ext_props
VkExtensionProperties ext_props
Definition: vulkan_decode.h:34
FFVulkanDecodePicture::ref_slot
VkVideoReferenceSlotInfoKHR ref_slot
Definition: vulkan_decode.h:101
FFVulkanDecodePicture::sem
VkSemaphore sem
Definition: vulkan_decode.h:96
FFVulkanDecodeContext::external_fg
int external_fg
Definition: vulkan_decode.h:72
FFVulkanDecodePicture::ref_slots
VkVideoReferenceSlotInfoKHR ref_slots[36]
Definition: vulkan_decode.h:105
FFVkExecPool
Definition: vulkan.h:210
FFVulkanDecodeContext::hevc_headers
struct HEVCHeaderSet * hevc_headers
Definition: vulkan_decode.h:80
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:186
AVCodecContext
main external API structure.
Definition: avcodec.h:445
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:70
FFVulkanDecodeDescriptor
Definition: vulkan_decode.h:29
FFVulkanDecodePicture::dpb_frame
AVFrame * dpb_frame
Definition: vulkan_decode.h:88
vulkan_video.h
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:111
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
FFVulkanExtensions
FFVulkanExtensions
Definition: vulkan_functions.h:29
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1107
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:312
FFVulkanDecodePicture::img_view_dest
VkImageView img_view_dest
Definition: vulkan_decode.h:92
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
FFVulkanDecodeContext::layered_dpb
int layered_dpb
Definition: vulkan_decode.h:71
FFVulkanDecodeShared::qf
FFVkQueueFamilyCtx qf
Definition: vulkan_decode.h:49
FFVulkanDecodePicture::destroy_image_view
PFN_vkDestroyImageView destroy_image_view
Definition: vulkan_decode.h:116
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:108
FFVulkanDecodeContext::quirk_av1_offset
int quirk_av1_offset
Definition: vulkan_decode.h:77
FFVulkanDecodePicture::img_aspect
VkImageAspectFlags img_aspect
Definition: vulkan_decode.h:93
FFVulkanDecodeShared::dec_caps
VkVideoDecodeCapabilitiesKHR dec_caps
Definition: vulkan_decode.h:52