FFmpeg滤镜代码级分析

前一篇文章《为FFmpeg添加自定义滤镜》详细讲述了FFmpeg的滤镜添加步骤，并给出了代码实例。

本文将以FFmpeg自带的deinterlace滤镜”yadif – yet another deinterlace filter”为例分析

FFmpeg滤镜的代码级实现机制。

总的来说,FFmpeg的滤镜机制和MicroSoft Directshow机制基本相同——不知道谁学了谁的，还是程

序员所见略同：

构建机制上：FFmpeg自己实现了统一的对象模型，DShow是COM的实现;

框架实现上：都使用filter graph来管理滤镜;

连接方式上：FFmpeg使用Pad结构体，DShow使用Pin，两者都是分配器的机制;

下面是以层级来分析代码：

FFmpeg转码层的伪码如下所示，红色粗体是和filter相关的函数:

int transcode(AVFormatContext **output_files,
              int nb_output_files,
              AVInputFile *input_files,
              int nb_input_files,
              AVStreamMap *stream_maps,
              int nb_stream_maps)
{
  …
  解码参数计算;


  #if CONFIG_AVFILTER
  if (configure_video_filters(ist, ost)) {
    fprintf(stderr, "Error opening filters!\n");
    ret = AVERROR(EINVAL);
    goto fail;
  }
  #endif


  编码参数计算;

  output_packet(ist, ist_index, ost_table,
                nb_ostreams, &pkt, &errorflag);

  ……

  return ret;
}

函数configure_video_filters是用来初始化filter graph 和filter本身的初始化；

函数output_packet是FFmpeg整个解码，滤镜处理，编码调用的实现。

1. 先来看configure_video_filters的实现：

int configure_video_filters(AVInputStream *ist,
                            AVOutputStream *ost)
{
   // 类似于directshow一样初始化filter graph等;
  ost->graph = avfilter_graph_alloc();
  …

    // 初始化filter graph和filter

  avfilter_graph_parse(ost->graph, ost->avfilter,
                       &inputs, &outputs, NULL)


  // 检查filter支持的媒体类型
  avfilter_graph_config(ost->graph, NULL);

  。。。

  return 0;
}

filter graph和filter的初始化:

int avfilter_graph_parse(AVFilterGraph *graph,
                         const char *filters,
                         AVFilterInOut **open_inputs,
                         AVFilterInOut **open_outputs,
                         void *log_ctx)
{

    do {

        parse_inputs(&filters, &curr_inputs, open_outputs, log_ctx);

        parse_filter(&filter, &filters, graph, index, log_ctx);


        if (filter->input_count == 1 && !curr_inputs && !index) {
            /* First input can be omitted if it is "[in]" */
            const char *tmp = "[in]";
            if ((ret = parse_inputs(&tmp, &curr_inputs, open_outputs,
                                    log_ctx)) < 0)
                goto fail;
        }


        link_filter_inouts(filter, &curr_inputs, open_inputs, log_ctx);


        parse_outputs(&filters, &curr_inputs, open_inputs,
                      open_outputs, log_ctx);


        filters += strspn(filters, WHITESPACES);
        chr = *filters++;

        index++;
    } while (chr == ',' || chr == ';');

    if (open_inputs && *open_inputs &&
        !strcmp((*open_inputs)->name, ""out"") && curr_inputs) {
        /* Last output can be omitted if it is "[out]" */
        const char *tmp = "[out]";
        if ((ret = parse_outputs(&tmp, &curr_inputs, open_inputs,
                                              open_outputs, log_ctx)) < 0)
            goto fail;

    }

    return 0;
}

static int parse_filter(AVFilterContext **filt_ctx,
                        const char **buf, AVFilterGraph *graph,"
                        int index, void *log_ctx)
{
    char *opts = NULL;
    char *name = av_get_token(buf, "=,;[\n");
    int ret;

    if (**buf == '=') {
        (*buf)++;
        opts = av_get_token(buf, "[],;\n");
    }

    ret = create_filter(filt_ctx, graph, index, name, opts, log_ctx);

    av_free(name);
    av_free(opts);
    return ret;
}

到了真正的Filter graph初始化，这部分和DShow很相似：

int create_filter(AVFilterContext **filt_ctx,
                  AVFilterGraph *ctx, int index,
                  const char *filt_name, const char *args,
                  void *log_ctx)

{
    AVFilter *filt;
    char inst_name[30];
    char tmp_args[256];
    int ret;

    // 注册当前filter到静态数组中
    filt = avfilter_get_by_name(filt_name);

    // 分配并初始化输入输出Pad
    ret = avfilter_open(filt_ctx, filt, inst_name);

    // 将当前filter添加到filter graph中
    ret = avfilter_graph_add_filter(ctx, *filt_ctx);

    // 通过函数指针，调用当前filter的初始化函数
    ret = avfilter_init_filter(*filt_ctx, args, NULL);
    return 0;
}

int avfilter_init_filter(AVFilterContext *filter,
                         const char *args,
                         void *opaque)
{
    int ret=0;

    if (filter->filter->init)
        ret = filter->filter->init(filter, args, opaque);
    return ret;
}

static av_cold int init(AVFilterContext *ctx,
                                  const char *args,
                                  void *opaque)
{
    YADIFContext *yadif = ctx->priv;
    av_unused int cpu_flags = av_get_cpu_flags();

    yadif->mode = 0;
    yadif->parity = -1;
    yadif->csp = NULL;

    // 滤镜函数指针初始化赋值
    yadif->filter_line = filter_line_c;

    return 0;
}

2. 接着来看两个filter的Pad连接时的媒体类型的协商机制

int avfilter_graph_config(AVFilterGraph *graphctx, void *log_ctx)
{
    int ret;

     // 检查filter的权限

    if ((ret = ff_avfilter_graph_check_validity(graphctx, log_ctx)))
        return ret;

    // 检查Pad支持的媒体类型
    if ((ret = ff_avfilter_graph_config_formats(graphctx, log_ctx)))
        return ret;

     // 

    if ((ret = ff_avfilter_graph_config_links(graphctx, log_ctx)))
        return ret;

    return 0;
}

检查Pad支持的媒体类型 :

int ff_avfilter_graph_config_formats(AVFilterGraph *graph,
                                                          AVClass *log_ctx)
{
    int ret;

    /* find supported formats from sub-filters, and merge along links */
    if ((ret = query_formats(graph, log_ctx)) < 0)
        return ret;

    /* Once everything is merged, it's possible that we'll still have
     * multiple valid media format choices. We pick the first one. */
    pick_formats(graph);

    return 0;
}

static int query_formats(AVFilterGraph *graph,
                         AVClass *log_ctx)
{
    int i, j, ret;
    int scaler_count = 0;
    char inst_name[30];

    /* ask all the sub-filters for their supported media formats */
    for (i = 0; i < graph->filter_count; i++) {
        if (graph->filters[i]->filter->query_formats)
            graph->filters[i]->filter->query_formats(graph->filters[i]);
        else
            avfilter_default_query_formats(graph->filters[i]);
    }

    /* go through and merge as many format lists as possible */
    for (i = 0; i < graph->filter_count; i++) {

       …

    }

    return 0;
}

/*  检查连接用Pad支持的媒体类型 */
static int query_formats(AVFilterContext *ctx)
{
    static const enum PixelFormat pix_fmts[] = {
        PIX_FMT_YUV420P,
        PIX_FMT_YUV422P,
        PIX_FMT_YUV444P,
        PIX_FMT_YUV410P,
        PIX_FMT_YUV411P,
        PIX_FMT_GRAY8,
        PIX_FMT_YUVJ420P,
        PIX_FMT_YUVJ422P,
        PIX_FMT_YUVJ444P,
        AV_NE( PIX_FMT_GRAY16BE, PIX_FMT_GRAY16LE ),
        PIX_FMT_YUV440P,
        PIX_FMT_YUVJ440P,
        AV_NE( PIX_FMT_YUV420P16BE, PIX_FMT_YUV420P16LE ),
        AV_NE( PIX_FMT_YUV422P16BE, PIX_FMT_YUV422P16LE ),
        AV_NE( PIX_FMT_YUV444P16BE, PIX_FMT_YUV444P16LE ),
        PIX_FMT_NONE
    };

    avfilter_set_common_pixel_formats(ctx,
                                      avfilter_make_format_list(pix_fmts));

    return 0;
}

3. 最后来看滤镜的连接和对每帧图像处理时的调用

int output_packet(AVInputStream *ist, int ist_index,
                  AVOutputStream **ost_table,
                  int nb_ostreams,
                  const AVPacket *pkt,
                  int *errorflag)
{

  /* decode the packet if needed */
  if (ist->decoding_needed) {

    解码;

#if CONFIG_AVFILTER
    if(ist->st->codec->codec_type == AVMEDIA_TYPE_VIDEO)
    if (start_time == 0 || ist->pts >= start_time) {
      for(i=0;i<nb_ostreams;i++) {
        ost = ost_table[i];
        if (ost->input_video_filter && ost->source_index == ist_index) {
          if (!picture.sample_aspect_ratio.num)
          picture.sample_aspect_ratio = ist->st->sample_aspect_ratio;
          picture.pts = ist->pts;

          av_vsrc_buffer_add_frame(ost->input_video_filter, &picture,                                                     AV_VSRC_BUF_FLAG_OVERWRITE);

        }
      }
    }
#endif

#if CONFIG_AVFILTER
    frame_available = ist->st->codec->codec_type
                      != AVMEDIA_TYPE_VIDEO ||
                      !ost->output_video_filter ||
                      avfilter_poll_frame(ost->output_video_filter->inputs[0]);
    while (frame_available) {
      if (ist->st->codec->codec_type == AVMEDIA_TYPE_VIDEO &&
          ost->output_video_filter) {
        AVRational ist_pts_tb = ost->output_video_filter->inputs[0]->time_base;
        if (av_vsink_buffer_get_video_buffer_ref(ost->output_video_filter,
            &ost->picref, 0) < 0)
          goto cont;
        if (ost->picref) {
          AVRational tempVar = {1, AV_TIME_BASE};
          avfilter_fill_frame_from_video_buffer_ref(&picture, ost->picref);
          //ist->pts = av_rescale_q(ost->picref->pts, ist_pts_tb, AV_TIME_BASE_Q);
          ist->pts = av_rescale_q(ost->picref->pts, ist_pts_tb, tempVar);//modify by chenrui
        }
      }
#endif

      编码;

      return 0;
      fail_decode:
      return -1;
   }
  }
}

先来看滤镜的连接：

int av_vsrc_buffer_add_frame(AVFilterContext *buffer_src,
                             const AVFrame *frame, int flags)
{
    // 从帧列表中得到当前帧
    AVFilterBufferRef *picref =
        avfilter_get_video_buffer_ref_from_frame(frame,
                                                 AV_PERM_WRITE);
    if (!picref)
        return AVERROR(ENOMEM);
    ret = av_vsrc_buffer_add_video_buffer_ref(buffer_src,
                                              picref, flags);


    picref->buf->data[0] = NULL;
    avfilter_unref_buffer(picref);

    return ret;
}


int av_vsrc_buffer_add_video_buffer_ref(AVFilterContext *buffer_filter,
                                        AVFilterBufferRef *picref, int flags)"
{

    智能插入相应的filter;

    c->picref = avfilter_get_video_buffer(outlink, AV_PERM_WRITE,
                                          picref->video->w, picref->video->h);

    av_image_copy(c->picref->data, c->picref->linesize,
                  picref->data, picref->linesize,
                  picref->format, picref->video->w, picref->video->h);
    avfilter_copy_buffer_ref_props(c->picref, picref);

    return 0;
}


AVFilterBufferRef *avfilter_get_video_buffer(AVFilterLink *link,
                                      int perms, int w, int h)
{
    AVFilterBufferRef *ret = NULL;

    if (link->dstpad->get_video_buffer)
        ret = link->dstpad->get_video_buffer(link, perms, w, h);

    if (!ret)
        ret = avfilter_default_get_video_buffer(link, perms, w, h);

    if (ret)
        ret->type = AVMEDIA_TYPE_VIDEO;

    return ret;
}

AVFilterBufferRef *get_video_buffer(AVFilterLink *link, int perms,
                                    int w, int h)
{
    AVFilterBufferRef *picref;
    int width = FFALIGN(w, 32);
    int height= FFALIGN(h+2, 32);
    int i;

    picref = avfilter_default_get_video_buffer(link, perms, width, height);

    picref->video->w = w;
    picref->video->h = h;

    for (i = 0; i < 3; i++)
        picref->data[i] += picref->linesize[i];

    return picref;
}

最后来看图像处理函数的实现：

int avfilter_poll_frame(AVFilterLink *link)
{
    int i, min = INT_MAX;

    if (link->srcpad->poll_frame)
        return link->srcpad->poll_frame(link);
    for (i = 0; i < link->src->input_count; i++) {
        int val;
        if (!link->src->inputs[i])
            return -1;
        val = avfilter_poll_frame(link->src->inputs[i]);
        min = FFMIN(min, val);
    }

    return min;
}


int poll_frame(AVFilterLink *link)
{
    YADIFContext *yadif = link->src->priv;
    int ret, val;

    if (yadif->frame_pending)
        return 1;

    val = avfilter_poll_frame(link->src->inputs[0]);

    if (val==1 && !yadif->next) {
      //FIXME change API to not requre this red tape
      if ((ret = avfilter_request_frame(link->src->inputs[0])) < 0)
        return ret;
      val = avfilter_poll_frame(link->src->inputs[0]);
    }
    assert(yadif->next || !val);

    return val * ((yadif->mode&1)+1);
}

int avfilter_request_frame(AVFilterLink *link)
{


    if (link->srcpad->request_frame)
        return link->srcpad->request_frame(link);
    else if (link->src->inputs[0])
        return avfilter_request_frame(link->src->inputs[0]);
    else return -1;
}



static int request_frame(AVFilterLink *link)
{
    BufferSourceContext *c = link->src->priv;

    if (!c->picref) {
        av_log(link->src, AV_LOG_WARNING,
               "request_frame() called with no available frame!\n");
        return AVERROR(EINVAL);
    }

     // 图像处理函数的真正实现

    avfilter_start_frame(link, avfilter_ref_buffer(c->picref, ~0));
    avfilter_draw_slice(link, 0, link->h, 1);
    avfilter_end_frame(link);

    avfilter_unref_buffer(c->picref);
    c->picref = NULL;

    return 0;
}

完结。