RVM-Inference
RVM-Inference copied to clipboard
DefTruth
请问如何将绿色背景替换为图片呢？
可以参考我在lite.ai.toolkit的一个回答：
- https://github.com/DefTruth/lite.ai.toolkit/issues/223
cv::Mat out_mat = fgr_mat.mul(three_channel_pha_mat) + bgr_mat.mul(1. - three_channel_pha_mat);
应该是用类似的逻辑，opencv的mul方法支持相同size大小的Mat进行逐元素运算。或者先将fgr_mat和img_bgr用split成三个通道，逐个通道运算后合并就行。
std::vector<cv::Mat> fgr_mat_channels, bgr_mat_channels;
cv::split(fgr_mat, fgr_mat_channels);
cv::split(img_bgr, bgr_mat_channels);
auto rmat = fgr_mat_channels.at(0);
auto bmat = fgr_mat_channels.at(1);
auto gmat = fgr_mat_channels.at(2);
auto brmat = bgr_mat_channels.at(0);
auto bbmat = bgr_mat_channels.at(1);
auto bgmat = bgr_mat_channels.at(2);
cv::Mat rest = 1. - pha_mat;
cv::Mat mbmat = bmat.mul(pmat) + rest.mul(brmat);
cv::Mat mgmat = gmat.mul(pmat) + rest.mul(bbmat);
cv::Mat mrmat = rmat.mul(pmat) + rest.mul(bgmat);
std::vector<cv::Mat> merge_channel_mats;
merge_channel_mats.push_back(mbmat);
merge_channel_mats.push_back(mgmat);
merge_channel_mats.push_back(mrmat);
cv::Mat merge_mat;
cv::merge(merge_channel_mats, merge_mat);
这是处理方式的问题。另外，后处理可以考虑增加保留最大连通区域的算法，去除一些小的黑点。这个后处理算法可以参考我的文章，就不这里解释原理了。
- 知乎文章：https://zhuanlan.zhihu.com/p/442949027
// https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L208
void MGMatting::remove_small_connected_area(cv::Mat &alpha_pred)
{
cv::Mat gray, binary;
alpha_pred.convertTo(gray, CV_8UC1, 255.f);
// 255 * 0.05 ~ 13
// https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L209
cv::threshold(gray, binary, 13, 255, cv::THRESH_BINARY);
// morphologyEx with OPEN operation to remove noise first.
auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(3, 3), cv::Point(-1, -1));
cv::morphologyEx(binary, binary, cv::MORPH_OPEN, kernel);
// Computationally connected domain
cv::Mat labels = cv::Mat::zeros(alpha_pred.size(), CV_32S);
cv::Mat stats, centroids;
int num_labels = cv::connectedComponentsWithStats(binary, labels, stats, centroids, 8, 4);
if (num_labels <= 1) return; // no noise, skip.
// find max connected area, 0 is background
int max_connected_id = 1; // 1,2,...
int max_connected_area = stats.at<int>(max_connected_id, cv::CC_STAT_AREA);
for (int i = 1; i < num_labels; ++i)
{
int tmp_connected_area = stats.at<int>(i, cv::CC_STAT_AREA);
if (tmp_connected_area > max_connected_area)
{
max_connected_area = tmp_connected_area;
max_connected_id = i;
}
}
const int h = alpha_pred.rows;
const int w = alpha_pred.cols;
// remove small connected area.
for (int i = 0; i < h; ++i)
{
int *label_row_ptr = labels.ptr<int>(i);
float *alpha_row_ptr = alpha_pred.ptr<float>(i);
for (int j = 0; j < w; ++j)
{
if (label_row_ptr[j] != max_connected_id)
alpha_row_ptr[j] = 0.f;
}
}
}
这个alpha_pred就是预测的pha_mat;
可以参考以下这段逻辑，我在lite里面增加了一些辅助函数，但还没合并进主分支，你可以参考下：
void lite::utils::swap_background(const cv::Mat &fgr_mat, const cv::Mat &pha_mat,
const cv::Mat &bgr_mat, cv::Mat &out_mat,
bool fgr_is_already_mul_pha)
{
// user-friendly method for background swap.
if (fgr_mat.empty() || pha_mat.empty() || bgr_mat.empty()) return;
const unsigned int fg_h = fgr_mat.rows;
const unsigned int fg_w = fgr_mat.cols;
const unsigned int bg_h = bgr_mat.rows;
const unsigned int bg_w = bgr_mat.cols;
const unsigned int ph_h = pha_mat.rows;
const unsigned int ph_w = pha_mat.cols;
const unsigned int channels = fgr_mat.channels();
if (channels != 3) return; // only support 3 channels.
const unsigned int num_elements = fg_h * fg_w * channels;
cv::Mat bg_mat_copy, ph_mat_copy, fg_mat_copy;
if (bg_h != fg_h || bg_w != fg_w)
cv::resize(bgr_mat, bg_mat_copy, cv::Size(fg_w, fg_h));
else bg_mat_copy = bgr_mat; // ref only.
if (ph_h != fg_h || ph_w != fg_w)
cv::resize(pha_mat, ph_mat_copy, cv::Size(fg_w, fg_h));
else ph_mat_copy = pha_mat; // ref only.
if (ph_mat_copy.channels() == 1)
cv::cvtColor(ph_mat_copy, ph_mat_copy, cv::COLOR_GRAY2BGR); // 0.~1.
// convert mats to float32 points.
if (bg_mat_copy.type() != CV_32FC3) bg_mat_copy.convertTo(bg_mat_copy, CV_32FC3); // 0.~255.
if (ph_mat_copy.type() != CV_32FC3) ph_mat_copy.convertTo(ph_mat_copy, CV_32FC3); // 0.~1.
if (fgr_mat.type() != CV_32FC3) fgr_mat.convertTo(fg_mat_copy, CV_32FC3); // 0.~255.
else fg_mat_copy = fgr_mat; // ref only
// element wise operations.
out_mat = fg_mat_copy.clone();
const float *fg_ptr = (float *) fg_mat_copy.data;
const float *bg_ptr = (float *) bg_mat_copy.data;
const float *ph_ptr = (float *) ph_mat_copy.data;
float *mutable_out_ptr = (float *) out_mat.data;
// TODO: add omp support instead of native loop.
if (!fgr_is_already_mul_pha)
for (unsigned int i = 0; i < num_elements; ++i)
mutable_out_ptr[i] = fg_ptr[i] * ph_ptr[i] + (1.f - ph_ptr[i]) * bg_ptr[i];
else
for (unsigned int i = 0; i < num_elements; ++i)
mutable_out_ptr[i] = fg_ptr[i] + (1.f - ph_ptr[i]) * bg_ptr[i];
if (!out_mat.empty() && out_mat.type() != CV_8UC3)
out_mat.convertTo(out_mat, CV_8UC3);
}
使用案例（MODNet还在开发中，此处仅用作参考示例）
static void test_default()
{
std::string onnx_path = "../../../hub/onnx/cv/modnet_photographic_portrait_matting-512x512.onnx";
std::string test_img_path = "../../../examples/lite/resources/test_lite_matting_input.jpg";
std::string test_bgr_path = "../../../examples/lite/resources/test_lite_matting_bgr.jpg";
std::string save_fgr_path = "../../../logs/test_lite_modnet_fgr.jpg";
std::string save_pha_path = "../../../logs/test_lite_modnet_pha.jpg";
std::string save_merge_path = "../../../logs/test_lite_modnet_merge.jpg";
std::string save_swap_path = "../../../logs/test_lite_modnet_swap.jpg";
lite::cv::matting::MODNet *modnet =
new lite::cv::matting::MODNet(onnx_path, 16); // 16 threads
lite::types::MattingContent content;
cv::Mat img_bgr = cv::imread(test_img_path);
cv::Mat bgr_mat = cv::imread(test_bgr_path);
// 1. image matting.
modnet->detect(img_bgr, content, true);
if (content.flag)
{
if (!content.fgr_mat.empty()) cv::imwrite(save_fgr_path, content.fgr_mat);
if (!content.pha_mat.empty()) cv::imwrite(save_pha_path, content.pha_mat * 255.);
if (!content.merge_mat.empty()) cv::imwrite(save_merge_path, content.merge_mat);
// swap background
cv::Mat out_mat;
lite::utils::swap_background(content.fgr_mat, content.pha_mat, bgr_mat, out_mat, true);
if (!out_mat.empty())
{
cv::imwrite(save_swap_path, out_mat);
std::cout << "Saved Swap Image Done!" << std::endl;
}
std::cout << "Default Version MGMatting Done!" << std::endl;
}
delete modnet;
}
效果示例
- 合成图
- 原图
- 背景图
