FFmpeg
vf_scale_vt.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2023 Zhao Zhili <zhilizhao@tencent.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 
23 #include "libavutil/hwcontext.h"
25 #include "libavutil/opt.h"
26 #include "libavutil/pixdesc.h"
27 #include "internal.h"
28 #include "scale_eval.h"
29 #include "video.h"
30 
31 typedef struct ScaleVtContext {
32  AVClass *class;
33 
34  VTPixelTransferSessionRef transfer;
37  char *w_expr;
38  char *h_expr;
39 
47 
49 {
50  ScaleVtContext *s = avctx->priv;
51  int ret;
52  CFStringRef value;
53 
54  ret = VTPixelTransferSessionCreate(kCFAllocatorDefault, &s->transfer);
55  if (ret != noErr) {
56  av_log(avctx, AV_LOG_ERROR, "transfer session create failed, %d\n", ret);
57  return AVERROR_EXTERNAL;
58  }
59 
60 #define STRING_OPTION(var_name, func_name, default_value) \
61  do { \
62  if (s->var_name##_string) { \
63  int var = av_##func_name##_from_name(s->var_name##_string); \
64  if (var < 0) { \
65  av_log(avctx, AV_LOG_ERROR, "Invalid %s.\n", #var_name); \
66  return AVERROR(EINVAL); \
67  } \
68  s->var_name = var; \
69  } else { \
70  s->var_name = default_value; \
71  } \
72  } while (0)
73 
75  STRING_OPTION(colour_transfer, color_transfer, AVCOL_TRC_UNSPECIFIED);
76  STRING_OPTION(colour_matrix, color_space, AVCOL_SPC_UNSPECIFIED);
77 
78  if (s->colour_primaries != AVCOL_PRI_UNSPECIFIED) {
80  if (!value) {
81  av_log(avctx, AV_LOG_ERROR,
82  "Doesn't support converting to colour primaries %s\n",
83  s->colour_primaries_string);
84  return AVERROR(ENOTSUP);
85  }
86  VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationColorPrimaries, value);
87  }
88 
89  if (s->colour_transfer != AVCOL_TRC_UNSPECIFIED) {
90  value = av_map_videotoolbox_color_trc_from_av(s->colour_transfer);
91  if (!value) {
92  av_log(avctx, AV_LOG_ERROR,
93  "Doesn't support converting to trc %s\n",
94  s->colour_transfer_string);
95  return AVERROR(ENOTSUP);
96  }
97  VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationTransferFunction, value);
98  }
99 
100  if (s->colour_matrix != AVCOL_SPC_UNSPECIFIED) {
102  if (!value) {
103  av_log(avctx, AV_LOG_ERROR,
104  "Doesn't support converting to colorspace %s\n",
105  s->colour_matrix_string);
106  return AVERROR(ENOTSUP);
107  }
108  VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationYCbCrMatrix, value);
109  }
110 
111  return 0;
112 }
113 
115 {
116  ScaleVtContext *s = avctx->priv;
117 
118  if (s->transfer) {
119  VTPixelTransferSessionInvalidate(s->transfer);
120  CFRelease(s->transfer);
121  s->transfer = NULL;
122  }
123 }
124 
126 {
127  int ret;
128  AVFilterContext *ctx = link->dst;
129  ScaleVtContext *s = ctx->priv;
130  AVFilterLink *outlink = ctx->outputs[0];
131  CVPixelBufferRef src;
132  CVPixelBufferRef dst;
133 
134  AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
135  if (!out) {
136  ret = AVERROR(ENOMEM);
137  goto fail;
138  }
139 
140  ret = av_frame_copy_props(out, in);
141  if (ret < 0)
142  goto fail;
143 
144  av_reduce(&out->sample_aspect_ratio.num, &out->sample_aspect_ratio.den,
145  (int64_t)in->sample_aspect_ratio.num * outlink->h * link->w,
146  (int64_t)in->sample_aspect_ratio.den * outlink->w * link->h,
147  INT_MAX);
148  if (s->colour_primaries != AVCOL_PRI_UNSPECIFIED)
149  out->color_primaries = s->colour_primaries;
150  if (s->colour_transfer != AVCOL_TRC_UNSPECIFIED)
151  out->color_trc = s->colour_transfer;
152  if (s->colour_matrix != AVCOL_SPC_UNSPECIFIED)
153  out->colorspace = s->colour_matrix;
154 
155  src = (CVPixelBufferRef)in->data[3];
156  dst = (CVPixelBufferRef)out->data[3];
157  ret = VTPixelTransferSessionTransferImage(s->transfer, src, dst);
158  if (ret != noErr) {
159  av_log(ctx, AV_LOG_ERROR, "transfer image failed, %d\n", ret);
161  goto fail;
162  }
163 
164  av_frame_free(&in);
165 
166  return ff_filter_frame(outlink, out);
167 
168 fail:
169  av_frame_free(&in);
170  av_frame_free(&out);
171  return ret;
172 }
173 
175 {
176  int err;
177  AVFilterContext *avctx = outlink->src;
178  ScaleVtContext *s = avctx->priv;
179  AVFilterLink *inlink = outlink->src->inputs[0];
180  AVHWFramesContext *hw_frame_ctx_in;
181  AVHWFramesContext *hw_frame_ctx_out;
182 
183  err = ff_scale_eval_dimensions(s, s->w_expr, s->h_expr, inlink, outlink,
184  &s->output_width,
185  &s->output_height);
186  if (err < 0)
187  return err;
188 
189  outlink->w = s->output_width;
190  outlink->h = s->output_height;
191 
192  if (inlink->sample_aspect_ratio.num) {
193  AVRational r = {outlink->h * inlink->w, outlink->w * inlink->h};
194  outlink->sample_aspect_ratio = av_mul_q(r, inlink->sample_aspect_ratio);
195  } else {
196  outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
197  }
198 
199  hw_frame_ctx_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
200 
201  av_buffer_unref(&outlink->hw_frames_ctx);
202  outlink->hw_frames_ctx = av_hwframe_ctx_alloc(hw_frame_ctx_in->device_ref);
203  hw_frame_ctx_out = (AVHWFramesContext *)outlink->hw_frames_ctx->data;
204  hw_frame_ctx_out->format = AV_PIX_FMT_VIDEOTOOLBOX;
205  hw_frame_ctx_out->sw_format = hw_frame_ctx_in->sw_format;
206  hw_frame_ctx_out->width = outlink->w;
207  hw_frame_ctx_out->height = outlink->h;
208 
209  err = ff_filter_init_hw_frames(avctx, outlink, 1);
210  if (err < 0)
211  return err;
212 
213  err = av_hwframe_ctx_init(outlink->hw_frames_ctx);
214  if (err < 0) {
215  av_log(avctx, AV_LOG_ERROR,
216  "Failed to init videotoolbox frame context, %s\n",
217  av_err2str(err));
218  return err;
219  }
220 
221  return 0;
222 }
223 
224 #define OFFSET(x) offsetof(ScaleVtContext, x)
225 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
226 static const AVOption scale_vt_options[] = {
227  { "w", "Output video width",
228  OFFSET(w_expr), AV_OPT_TYPE_STRING, {.str = "iw"}, .flags = FLAGS },
229  { "h", "Output video height",
230  OFFSET(h_expr), AV_OPT_TYPE_STRING, {.str = "ih"}, .flags = FLAGS },
231  { "color_matrix", "Output colour matrix coefficient set",
232  OFFSET(colour_matrix_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
233  { "color_primaries", "Output colour primaries",
234  OFFSET(colour_primaries_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
235  { "color_transfer", "Output colour transfer characteristics",
236  OFFSET(colour_transfer_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
237  { NULL },
238 };
239 
240 AVFILTER_DEFINE_CLASS(scale_vt);
241 
242 static const AVFilterPad scale_vt_inputs[] = {
243  {
244  .name = "default",
245  .type = AVMEDIA_TYPE_VIDEO,
246  .filter_frame = &scale_vt_filter_frame,
247  },
248 };
249 
250 static const AVFilterPad scale_vt_outputs[] = {
251  {
252  .name = "default",
253  .type = AVMEDIA_TYPE_VIDEO,
254  .config_props = &scale_vt_config_output,
255  },
256 };
257 
259  .name = "scale_vt",
260  .description = NULL_IF_CONFIG_SMALL("Scale Videotoolbox frames"),
261  .priv_size = sizeof(ScaleVtContext),
262  .init = scale_vt_init,
267  .priv_class = &scale_vt_class,
268  .flags = AVFILTER_FLAG_HWDEVICE,
269  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
270 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:108
av_map_videotoolbox_color_trc_from_av
CFStringRef av_map_videotoolbox_color_trc_from_av(enum AVColorTransferCharacteristic trc)
Convert an AVColorTransferCharacteristic to a VideoToolbox/CoreVideo color transfer function string.
Definition: hwcontext_videotoolbox.c:463
r
const char * r
Definition: vf_curves.c:126
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
AVColorTransferCharacteristic
AVColorTransferCharacteristic
Color Transfer Characteristic.
Definition: pixfmt.h:570
out
FILE * out
Definition: movenc.c:54
av_map_videotoolbox_color_matrix_from_av
CFStringRef av_map_videotoolbox_color_matrix_from_av(enum AVColorSpace space)
Convert an AVColorSpace to a VideoToolbox/CoreVideo color matrix string.
Definition: hwcontext_videotoolbox.c:411
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: internal.h:364
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:978
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
AVHWFramesContext::format
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
Definition: hwcontext.h:209
ff_vf_scale_vt
const AVFilter ff_vf_scale_vt
Definition: vf_scale_vt.c:258
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:334
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
pixdesc.h
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:248
AVOption
AVOption.
Definition: opt.h:251
AVCOL_TRC_UNSPECIFIED
@ AVCOL_TRC_UNSPECIFIED
Definition: pixfmt.h:573
ff_scale_eval_dimensions
int ff_scale_eval_dimensions(void *log_ctx, const char *w_expr, const char *h_expr, AVFilterLink *inlink, AVFilterLink *outlink, int *ret_w, int *ret_h)
Parse and evaluate string expressions for width and height.
Definition: scale_eval.c:57
AVColorPrimaries
AVColorPrimaries
Chromaticity coordinates of the source primaries.
Definition: pixfmt.h:545
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:229
video.h
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:361
scale_vt_filter_frame
static int scale_vt_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_scale_vt.c:125
FLAGS
#define FLAGS
Definition: vf_scale_vt.c:225
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:412
fail
#define fail()
Definition: checkasm.h:138
scale_vt_init
static av_cold int scale_vt_init(AVFilterContext *avctx)
Definition: vf_scale_vt.c:48
av_reduce
int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
Reduce a fraction.
Definition: rational.c:35
AVRational::num
int num
Numerator.
Definition: rational.h:59
OFFSET
#define OFFSET(x)
Definition: vf_scale_vt.c:224
scale_vt_config_output
static int scale_vt_config_output(AVFilterLink *outlink)
Definition: vf_scale_vt.c:174
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:47
av_map_videotoolbox_color_primaries_from_av
CFStringRef av_map_videotoolbox_color_primaries_from_av(enum AVColorPrimaries pri)
Convert an AVColorPrimaries to a VideoToolbox/CoreVideo color primaries string.
Definition: hwcontext_videotoolbox.c:438
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
AVHWFramesContext::height
int height
Definition: hwcontext.h:229
s
#define s(width, name)
Definition: cbs_vp9.c:198
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts_bsf.c:365
ctx
AVFormatContext * ctx
Definition: movenc.c:48
AVCOL_PRI_UNSPECIFIED
@ AVCOL_PRI_UNSPECIFIED
Definition: pixfmt.h:548
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:192
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:222
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:736
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
ScaleVtContext::output_width
int output_width
Definition: vf_scale_vt.c:35
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
AVHWFramesContext::device_ref
AVBufferRef * device_ref
A reference to the parent AVHWDeviceContext.
Definition: hwcontext.h:141
AVFilterContext::inputs
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:405
ScaleVtContext::colour_primaries_string
char * colour_primaries_string
Definition: vf_scale_vt.c:43
ScaleVtContext::w_expr
char * w_expr
Definition: vf_scale_vt.c:37
color_primaries
static const AVColorPrimariesDesc color_primaries[AVCOL_PRI_NB]
Definition: csp.c:76
hwcontext_videotoolbox.h
ScaleVtContext::colour_primaries
enum AVColorPrimaries colour_primaries
Definition: vf_scale_vt.c:40
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:106
ScaleVtContext::h_expr
char * h_expr
Definition: vf_scale_vt.c:38
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:121
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:138
ScaleVtContext::transfer
VTPixelTransferSessionRef transfer
Definition: vf_scale_vt.c:34
scale_eval.h
ScaleVtContext::output_height
int output_height
Definition: vf_scale_vt.c:36
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
scale_vt_uninit
static av_cold void scale_vt_uninit(AVFilterContext *avctx)
Definition: vf_scale_vt.c:114
internal.h
ScaleVtContext::colour_transfer_string
char * colour_transfer_string
Definition: vf_scale_vt.c:44
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:182
AV_PIX_FMT_VIDEOTOOLBOX
@ AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
Definition: pixfmt.h:302
AVColorSpace
AVColorSpace
YUV colorspace type.
Definition: pixfmt.h:599
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:53
scale_vt_options
static const AVOption scale_vt_options[]
Definition: vf_scale_vt.c:226
AVCOL_SPC_UNSPECIFIED
@ AVCOL_SPC_UNSPECIFIED
Definition: pixfmt.h:602
scale_vt_outputs
static const AVFilterPad scale_vt_outputs[]
Definition: vf_scale_vt.c:250
ScaleVtContext::colour_transfer
enum AVColorTransferCharacteristic colour_transfer
Definition: vf_scale_vt.c:41
AVFilter
Filter definition.
Definition: avfilter.h:166
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:124
ret
ret
Definition: filter_design.txt:187
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:447
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(scale_vt)
AVRational::den
int den
Denominator.
Definition: rational.h:60
scale_vt_inputs
static const AVFilterPad scale_vt_inputs[]
Definition: vf_scale_vt.c:242
av_mul_q
AVRational av_mul_q(AVRational b, AVRational c)
Multiply two rationals.
Definition: rational.c:80
AVFilterContext
An instance of a filter.
Definition: avfilter.h:397
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
STRING_OPTION
#define STRING_OPTION(var_name, func_name, default_value)
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:193
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
hwcontext.h
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
uninit
static av_cold int uninit(AVCodecContext *avctx)
Definition: crystalhd.c:285
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
ScaleVtContext::colour_matrix
enum AVColorSpace colour_matrix
Definition: vf_scale_vt.c:42
ScaleVtContext::colour_matrix_string
char * colour_matrix_string
Definition: vf_scale_vt.c:45
ff_filter_init_hw_frames
int ff_filter_init_hw_frames(AVFilterContext *avctx, AVFilterLink *link, int default_pool_size)
Perform any additional setup required for hardware frames.
Definition: avfilter.c:1547
ScaleVtContext
Definition: vf_scale_vt.c:31