Flutter Windows 渲染外部纹理

Flutter 支持通过在 native 侧注册一个本地纹理来将 RGBA8 格式的外部图像绘制到 TextureWidget 内。因此这个功能特别适合同离屏渲染技术结合来嵌入原本 native 侧才能渲染的内容，比如视频图像、游戏画面。

理论上这种方法会耗费大量的资源，因为经过了从 GPU(OpenGL) -> CPU(PixelBuffer) -> GPU(flutter) 的过程，在高分辨率、高帧率的情况下性能一定是不理想的。但是在本文写作的时间目前，Flutter Windows 暂时不支持共享 OpenGL context 以及 PlatformView，因此这是目前唯一的选择。
​

首先由插件在初始化时获取一个flutter::TextureRegistrar对象，并在新的帧到来后调用该对象上的MarkTextureFrameAvailable方法，触发 flutter 重绘。Flutter 引擎在状态改变，或者由于前面的回调触发进行重绘时，会调用由 native 侧事先注册的回调函数以获取一个 RGBA8 格式的 pixel buffer. 但需要注意的是应当避免在回调中进行耗时的渲染操作，而是在后台线程准备好缓冲区内容后，在回调中回传缓冲区指针即可。
​

下面以渲染 mpv 播放器的视频帧到 flutter 控件内为例。首先实现一个单独的渲染线程，并在该线程中初始化好 opengl 环境。在离屏渲染中，我们需要创建一个隐藏的窗体，并准备好一个 Framebuffer Object（FBO）。在 mpv 绘制帧数据到 FBO 后，通过 glReadPixels获得对应的 RGBA8 缓冲。

#pragma once
#pragma warning(disable : 4505)

#include <atomic>
#include <functional>
#include <iostream>
#include <memory>
#include <thread>

#include "common/GL/glew.h"
#include "common/GL/glfw3.h"
#include "common/mpv_controller.h"
#include "common/semaphore.h"
#include "common/buffer.h"

static void* get_proc_address(void* ctx, const char* name) {
  void* p = (void*)wglGetProcAddress(name);
  if (p == 0 || (p == (void*)0x1) || (p == (void*)0x2) || (p == (void*)0x3) ||
      (p == (void*)-1)) {
    HMODULE module = LoadLibraryA("opengl32.dll");
    p = (void*)GetProcAddress(module, name);
  }

  return p;
}

static void glfw_error_callback(int error, const char* desc) {
  LOG(INFO) << desc;
}

using RenderCb = std::function<void(void)>;

class RenderThread {
  std::shared_ptr<Semaphore> render_trigger;
  std::atomic_bool quit{false};
  std::thread loop;

  // opengl entries
  GLFWwindow* osr_window = nullptr;
  GLuint fbo = 1;
  GLuint texture;
  GLuint depth_render_buffer;
  GLuint color_render_buffer;

  // callbacks
  RenderCb render_callback;

  /// Called by mpv to invoke a new call to render
  static void mpv_frame_callback(void* ctx) {
    if (!ctx) {
      return;
    }
    auto* render_thread = static_cast<RenderThread*>(ctx);
    render_thread->render_trigger->signal();
  }

 public:
  RenderThread();
  ~RenderThread();

  std::atomic_bool started = false;

  /// Start render loop
  void start_render(std::shared_ptr<BufferController> buffer_controller,
                    RenderCb _render_callback) {
    if (!_render_callback || !buffer_controller) return;

    this->render_callback = _render_callback;
    quit = false;
    loop = std::thread([=]() {
      LOG(INFO) << "Render thread started";

      if (!glfwInit()) {
        LOG(FATAL) << "Init glfw failed";
        return;
      }
      glfwSetErrorCallback(glfw_error_callback);
      glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
      osr_window = glfwCreateWindow(1, 1, "", nullptr, nullptr);
      if (!osr_window) {
        LOG(FATAL) << "Init glfw window failed";
        return;
      }
      glfwMakeContextCurrent(osr_window);
      LOG(INFO) << "Finish init glfw";

      // glew
      glewExperimental = TRUE;
      GLenum err = glewInit();
      if (err != GLEW_OK) {
        LOG(FATAL) << "GLEW init failed";
        return;
      }
      if (GLEW_EXT_framebuffer_object != GL_TRUE) {
        LOG(FATAL) << "FBO unavaliable";
        return;
      }

      // init mpv & gl
      auto* mpv = MpvController::instance()->mpv;
      LOG(INFO) << "Init mpv gl";
      mpv_opengl_init_params gl_init_params{get_proc_address, nullptr, nullptr};
      mpv_render_param params[]{
          {MPV_RENDER_PARAM_API_TYPE,
           const_cast<char*>(MPV_RENDER_API_TYPE_OPENGL)},
          {MPV_RENDER_PARAM_OPENGL_INIT_PARAMS, &gl_init_params},
          {MPV_RENDER_PARAM_INVALID, nullptr}};

      mpv_render_context* mpv_ctx;
      if (mpv_render_context_create(&mpv_ctx, mpv, params) < 0) {
        LOG(FATAL) << "Create mpv ractx failed";
        throw std::runtime_error("failed to initialize mpv GL context");
      }
      LOG(INFO) << "Init mpv gl finished";

      mpv_render_context_set_update_callback(
          mpv_ctx, &RenderThread::mpv_frame_callback, static_cast<void*>(this));
      MpvController::instance()->mpv_ctx = mpv_ctx;

      // init framebuffer
      glGenFramebuffers(1, &fbo);
      glBindFramebuffer(GL_FRAMEBUFFER, fbo);

      GLuint texture;
      glGenTextures(1, &texture);
      glBindTexture(GL_TEXTURE_2D, texture);
      glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, 800, 400, 0, GL_RGBA,
                   GL_UNSIGNED_BYTE, nullptr);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
      glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
                             GL_TEXTURE_2D, texture, 0);

      err = glCheckFramebufferStatus(GL_FRAMEBUFFER);
      if (err != GL_FRAMEBUFFER_COMPLETE) {
        LOG(FATAL) << "FBO imcomplete";
        return;
      }

      // render loop
      started = true;
      LOG(INFO) << "Render loop started";
      while (true) {
        render_trigger->wait();
        if (quit) {
          break;
        }

        // perform render
        mpv_opengl_fbo mpfbo{static_cast<int>(fbo), 800, 400, 0};
        int flip_y = 0;

        mpv_render_param render_params[] = {
            {MPV_RENDER_PARAM_OPENGL_FBO, &mpfbo},
            {MPV_RENDER_PARAM_FLIP_Y, &flip_y},
            {MPV_RENDER_PARAM_INVALID, nullptr}};
        glClearColor(0, 0, 0, 0);
        glClear(GL_COLOR_BUFFER_BIT);
        mpv_render_context_render(MpvController::instance()->mpv_ctx,
                                  render_params);

        auto render_buffer = buffer_controller->get_render();
        render_buffer->reconfig(800, 400);
        glBindFramebuffer(GL_FRAMEBUFFER, fbo);
        glBindFramebuffer(GL_READ_BUFFER, fbo);
        glReadBuffer(GL_COLOR_ATTACHMENT0);
        glReadPixels(0, 0, 800, 400, GL_RGBA, GL_UNSIGNED_BYTE,
                     render_buffer->buffer);

        {
          GLenum glerr;
          while ((glerr = glGetError()) != GL_NO_ERROR) {
            LOG(DEBUG) << "GL error:" << glerr;
          }
        }
        buffer_controller->release_render(render_buffer);
        LOG(INFO) << "New mpv frame rendered";
        render_callback();
      }
      LOG(INFO) << "Render loop end";
      started = false;
      quit = false;
      glfwTerminate();
    });
  }
};

RenderThread::RenderThread() {
  render_trigger = std::make_shared<Semaphore>(0);
}

RenderThread::~RenderThread() {
  // quit render thread
  quit = true;
  render_trigger->signal();
  loop.join();
}

为了解决 mpv 渲染（生产者）和 flutter 渲染（消费者）两个线程的异步问题，我们需要随手实现一个多缓冲 buffer。两条额外的蓝色线分别对应生产者 overflow 和 underflow 的情况。

#include "buffer.h"
#include "easylogging++.h"

BufferController::BufferController(int buffer_count) {
  auto count = buffer_count < 3 ? 3 : buffer_count;
  for (int i = 0; i < count; ++i) {
    dirty_queue.emplace_back(
        std::make_shared<MpvRenderBuffer>());  // not a valid buffer currently
  }
}

BufferController::~BufferController() {}

SharedBuffer BufferController::get_render() {
  std::lock_guard lock(mu);
  auto& target_buffer = (!dirty_queue.empty()) ? dirty_queue : ready_queue;
  if (target_buffer.empty()) {
    return nullptr;
  }

  auto render_target = target_buffer.front();
  target_buffer.pop_front();
  return render_target;
}

void BufferController::release_render(SharedBuffer& buffer) {
  std::lock_guard lock(mu);
  ready_queue.push_back(buffer);
}

SharedBuffer BufferController::get_use() {
  std::lock_guard lock(mu);
  if (!ready_queue.empty()) {
    auto use_target = ready_queue.front();
    ready_queue.pop_front();
    return use_target;
  } else if (!dirty_queue.empty()) {
    // reused last buffer
    auto use_target = dirty_queue.back();
    dirty_queue.pop_back();
    return use_target;
  }
  return nullptr;
}

void BufferController::release_use(SharedBuffer& buffer) {
  std::lock_guard lock(mu);
  dirty_queue.push_back(buffer);
}

标签：include,render,Windows,void,纹理,mpv,buffer,GL,Flutter
来源： https://www.cnblogs.com/rockeyboard/p/15866770.html