OpenShot Library | libopenshot  0.7.0
ObjectMask.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2026 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include "effects/ObjectMask.h"
14 
15 #include "Exceptions.h"
16 #include "Frame.h"
17 #include "objdetectdata.pb.h"
18 
19 #define int64 opencv_broken_int
20 #define uint64 opencv_broken_uint
21 #include <opencv2/core.hpp>
22 #include <opencv2/imgproc.hpp>
23 #undef uint64
24 #undef int64
25 
26 #include <QColor>
27 #include <QImage>
28 #include <QPainter>
29 
30 #include <algorithm>
31 #include <fstream>
32 
33 using namespace openshot;
34 
35 namespace {
36 
37 QImage AlphaMaskImageFromRLE(const ObjectMaskFrameData& mask)
38 {
39  QImage image(mask.width, mask.height, QImage::Format_ARGB32_Premultiplied);
40  image.fill(Qt::transparent);
41  if (!mask.HasData())
42  return image;
43 
44  QRgb* data = reinterpret_cast<QRgb*>(image.bits());
45  const int total = mask.width * mask.height;
46  int offset = 0;
47  bool value = false;
48  for (uint32_t count : mask.rle) {
49  const int end = std::min(total, offset + static_cast<int>(count));
50  if (value)
51  std::fill(data + offset, data + end, qRgba(255, 255, 255, 255));
52  offset = end;
53  value = !value;
54  if (offset >= total)
55  break;
56  }
57  return image;
58 }
59 
60 cv::Mat BinaryMaskFromImage(const QImage& image)
61 {
62  QImage rgba = image.convertToFormat(QImage::Format_RGBA8888);
63  cv::Mat binary(rgba.height(), rgba.width(), CV_8UC1, cv::Scalar(0));
64  for (int y = 0; y < rgba.height(); ++y) {
65  const uchar* source = rgba.constScanLine(y);
66  uchar* target = binary.ptr<uchar>(y);
67  for (int x = 0; x < rgba.width(); ++x)
68  target[x] = source[x * 4 + 3] > 0 ? 255 : 0;
69  }
70  return binary;
71 }
72 
73 QImage StrokeImageFromMask(const QImage& alphaMask, int width)
74 {
75  QImage result(alphaMask.size(), QImage::Format_ARGB32_Premultiplied);
76  result.fill(Qt::transparent);
77  if (width <= 0)
78  return result;
79 
80  cv::Mat binary = BinaryMaskFromImage(alphaMask);
81  cv::Mat dilated;
82  const int kernelSize = std::max(1, width * 2 + 1);
83  cv::Mat kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(kernelSize, kernelSize));
84  cv::dilate(binary, dilated, kernel);
85  cv::Mat edge = dilated - binary;
86 
87  for (int y = 0; y < edge.rows; ++y) {
88  const uchar* edgeRow = edge.ptr<uchar>(y);
89  QRgb* target = reinterpret_cast<QRgb*>(result.scanLine(y));
90  for (int x = 0; x < edge.cols; ++x) {
91  if (edgeRow[x])
92  target[x] = qRgba(255, 255, 255, 255);
93  }
94  }
95  return result;
96 }
97 
98 }
99 
100 ObjectMask::ObjectMask()
101  : draw_mask(1.0)
102  , mask_color(83, 160, 237, 255)
103  , mask_alpha(120.0 / 255.0)
104  , stroke_color(255, 255, 255, 255)
105  , stroke_alpha(1.0)
106  , stroke_width(3.0)
107 {
108  init_effect_details();
109 }
110 
111 void ObjectMask::init_effect_details()
112 {
113  InitEffectInfo();
114  info.class_name = "ObjectMask";
115  info.name = "Object Mask";
116  info.description = "Create and draw a segmentation mask for a prompted object.";
117  info.has_audio = false;
118  info.has_video = true;
119  info.has_tracked_object = true;
120 }
121 
122 std::shared_ptr<Frame> ObjectMask::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
123 {
124  std::shared_ptr<QImage> frame_image = frame->GetImage();
125  if (!frame_image || frame_image->isNull() || draw_mask.GetValue(frame_number) != 1)
126  return frame;
127 
128  auto mask_it = masksData.find(frame_number);
129  if (mask_it == masksData.end() || !mask_it->second.HasData())
130  return frame;
131 
132  QImage alpha_mask = AlphaMaskImageFromRLE(mask_it->second)
133  .scaled(frame_image->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
134  std::vector<int> mask_rgba = mask_color.GetColorRGBA(frame_number);
135  QColor overlay_color(mask_rgba[0], mask_rgba[1], mask_rgba[2], 255 * mask_alpha.GetValue(frame_number));
136 
137  QImage overlay(frame_image->size(), QImage::Format_ARGB32_Premultiplied);
138  overlay.fill(Qt::transparent);
139  QPainter overlay_painter(&overlay);
140  overlay_painter.setCompositionMode(QPainter::CompositionMode_Source);
141  overlay_painter.fillRect(overlay.rect(), overlay_color);
142  overlay_painter.setCompositionMode(QPainter::CompositionMode_DestinationIn);
143  overlay_painter.drawImage(0, 0, alpha_mask);
144  overlay_painter.end();
145 
146  QPainter painter(frame_image.get());
147  painter.drawImage(0, 0, overlay);
148 
149  const int strokeWidth = static_cast<int>(std::round(stroke_width.GetValue(frame_number)));
150  if (strokeWidth > 0 && stroke_alpha.GetValue(frame_number) > 0.0) {
151  QImage stroke_mask = StrokeImageFromMask(alpha_mask, strokeWidth);
152  std::vector<int> stroke_rgba = stroke_color.GetColorRGBA(frame_number);
153  QColor stroke_qcolor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha.GetValue(frame_number));
154 
155  QImage stroke_overlay(frame_image->size(), QImage::Format_ARGB32_Premultiplied);
156  stroke_overlay.fill(Qt::transparent);
157  QPainter stroke_painter(&stroke_overlay);
158  stroke_painter.setCompositionMode(QPainter::CompositionMode_Source);
159  stroke_painter.fillRect(stroke_overlay.rect(), stroke_qcolor);
160  stroke_painter.setCompositionMode(QPainter::CompositionMode_DestinationIn);
161  stroke_painter.drawImage(0, 0, stroke_mask);
162  stroke_painter.end();
163  painter.drawImage(0, 0, stroke_overlay);
164  }
165  painter.end();
166 
167  return frame;
168 }
169 
170 bool ObjectMask::LoadObjMaskData(std::string inputFilePath)
171 {
172  pb_objdetect::ObjDetect objMessage;
173  std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
174  if (!objMessage.ParseFromIstream(&input))
175  return false;
176 
177  masksData.clear();
178  trackedObjects.clear();
179 
180  auto trackedObject = std::make_shared<TrackedObjectBBox>(83, 160, 237, 255);
181  trackedObject->Id(Id().empty() ? "Object Mask" : Id() + "-1");
182  trackedObject->ParentClip(this->ParentClip());
183  trackedObject->draw_box = Keyframe(0.0);
184  trackedObject->draw_text = Keyframe(0.0);
185  trackedObject->draw_mask = draw_mask;
186  trackedObject->mask_alpha = mask_alpha;
187  trackedObject->mask_color = mask_color;
188  trackedObject->stroke = stroke_color;
189  trackedObject->stroke_alpha = stroke_alpha;
190  trackedObject->stroke_width = stroke_width;
191 
192  for (int frameIndex = 0; frameIndex < objMessage.frame_size(); ++frameIndex) {
193  const auto& pbFrame = objMessage.frame(frameIndex);
194  if (pbFrame.bounding_box_size() <= 0)
195  continue;
196 
197  const auto& box = pbFrame.bounding_box(0);
198  ObjectMaskFrameData mask;
199  mask.box = BBox(box.x() + box.w() / 2.0f, box.y() + box.h() / 2.0f, box.w(), box.h(), 0.0f);
200  mask.score = box.confidence();
201  if (box.has_mask()) {
202  mask.width = box.mask().width();
203  mask.height = box.mask().height();
204  for (int rleIndex = 0; rleIndex < box.mask().rle_size(); ++rleIndex)
205  mask.rle.push_back(box.mask().rle(rleIndex));
206  }
207  masksData[pbFrame.id()] = mask;
208 
209  trackedObject->AddBox(pbFrame.id(), mask.box.cx, mask.box.cy, mask.box.width, mask.box.height, 0.0f);
210  if (mask.HasData()) {
211  ObjectMaskData trackedMask;
212  trackedMask.width = mask.width;
213  trackedMask.height = mask.height;
214  trackedMask.rle = mask.rle;
215  trackedObject->AddMask(pbFrame.id(), trackedMask);
216  }
217  }
218 
219  if (!masksData.empty())
220  trackedObjects[1] = trackedObject;
221 
222  google::protobuf::ShutdownProtobufLibrary();
223  return true;
224 }
225 
226 std::shared_ptr<QImage> ObjectMask::TrackedObjectMask(std::shared_ptr<QImage> target_image, int64_t frame_number) const
227 {
228  if (!target_image || target_image->isNull())
229  return {};
230 
231  auto mask_it = masksData.find(frame_number);
232  if (mask_it == masksData.end() || !mask_it->second.HasData())
233  return {};
234 
235  QImage alpha_mask = AlphaMaskImageFromRLE(mask_it->second)
236  .scaled(target_image->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
237 
238  auto mask_image = std::make_shared<QImage>(
239  target_image->width(), target_image->height(), QImage::Format_RGBA8888_Premultiplied);
240  mask_image->fill(QColor(0, 0, 0, 255));
241  QPainter painter(mask_image.get());
242  painter.drawImage(0, 0, alpha_mask);
243  painter.end();
244  return mask_image;
245 }
246 
247 std::string ObjectMask::Json() const
248 {
249  return JsonValue().toStyledString();
250 }
251 
252 Json::Value ObjectMask::JsonValue() const
253 {
254  Json::Value root = EffectBase::JsonValue();
255  root["type"] = info.class_name;
256  root["protobuf_data_path"] = protobuf_data_path;
257  root["draw_mask"] = draw_mask.JsonValue();
258  root["mask_color"] = mask_color.JsonValue();
259  root["mask_alpha"] = mask_alpha.JsonValue();
260  root["stroke_color"] = stroke_color.JsonValue();
261  root["stroke_alpha"] = stroke_alpha.JsonValue();
262  root["stroke_width"] = stroke_width.JsonValue();
263  return root;
264 }
265 
266 void ObjectMask::SetJson(const std::string value)
267 {
268  try {
269  SetJsonValue(openshot::stringToJson(value));
270  } catch (const std::exception&) {
271  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
272  }
273 }
274 
275 void ObjectMask::SetJsonValue(const Json::Value root)
276 {
277  EffectBase::SetJsonValue(root);
278 
279  if (!root["draw_mask"].isNull())
280  draw_mask.SetJsonValue(root["draw_mask"]);
281  if (!root["mask_color"].isNull())
282  mask_color.SetJsonValue(root["mask_color"]);
283  if (!root["mask_alpha"].isNull())
284  mask_alpha.SetJsonValue(root["mask_alpha"]);
285  if (!root["stroke_color"].isNull())
286  stroke_color.SetJsonValue(root["stroke_color"]);
287  if (!root["stroke"].isNull())
288  stroke_color.SetJsonValue(root["stroke"]);
289  if (!root["stroke_alpha"].isNull())
290  stroke_alpha.SetJsonValue(root["stroke_alpha"]);
291  if (!root["stroke_width"].isNull())
292  stroke_width.SetJsonValue(root["stroke_width"]);
293 
294  if (!root["protobuf_data_path"].isNull()) {
295  std::string new_path = root["protobuf_data_path"].asString();
296  if (protobuf_data_path != new_path || masksData.empty()) {
297  protobuf_data_path = new_path;
298  if (!LoadObjMaskData(protobuf_data_path))
299  throw InvalidFile("Invalid object mask protobuf data path", "");
300  }
301  }
302 }
303 
304 std::string ObjectMask::PropertiesJSON(int64_t requested_frame) const
305 {
306  Json::Value root = BasePropertiesJSON(requested_frame);
307  root["protobuf_data_path"] = add_property_json("Object Mask Data", 0.0, "string", protobuf_data_path, NULL, -1, -1, false, requested_frame);
308 
309  root["draw_mask"] = add_property_json("Draw Mask", draw_mask.GetValue(requested_frame), "int", "", &draw_mask, 0, 1, false, requested_frame);
310  root["draw_mask"]["choices"].append(add_property_choice_json("Yes", true, draw_mask.GetValue(requested_frame)));
311  root["draw_mask"]["choices"].append(add_property_choice_json("No", false, draw_mask.GetValue(requested_frame)));
312 
313  root["mask_color"] = add_property_json("Mask Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame);
314  root["mask_color"]["red"] = add_property_json("Red", mask_color.red.GetValue(requested_frame), "float", "", &mask_color.red, 0, 255, false, requested_frame);
315  root["mask_color"]["blue"] = add_property_json("Blue", mask_color.blue.GetValue(requested_frame), "float", "", &mask_color.blue, 0, 255, false, requested_frame);
316  root["mask_color"]["green"] = add_property_json("Green", mask_color.green.GetValue(requested_frame), "float", "", &mask_color.green, 0, 255, false, requested_frame);
317  root["mask_alpha"] = add_property_json("Mask Alpha", mask_alpha.GetValue(requested_frame), "float", "", &mask_alpha, 0.0, 1.0, false, requested_frame);
318 
319  root["stroke_color"] = add_property_json("Stroke Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame);
320  root["stroke_color"]["red"] = add_property_json("Red", stroke_color.red.GetValue(requested_frame), "float", "", &stroke_color.red, 0, 255, false, requested_frame);
321  root["stroke_color"]["blue"] = add_property_json("Blue", stroke_color.blue.GetValue(requested_frame), "float", "", &stroke_color.blue, 0, 255, false, requested_frame);
322  root["stroke_color"]["green"] = add_property_json("Green", stroke_color.green.GetValue(requested_frame), "float", "", &stroke_color.green, 0, 255, false, requested_frame);
323  root["stroke_alpha"] = add_property_json("Stroke Alpha", stroke_alpha.GetValue(requested_frame), "float", "", &stroke_alpha, 0.0, 1.0, false, requested_frame);
324  root["stroke_width"] = add_property_json("Stroke Width", stroke_width.GetValue(requested_frame), "int", "", &stroke_width, 0, 50, false, requested_frame);
325 
326  return root.toStyledString();
327 }
openshot::ClipBase::add_property_json
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
Definition: ClipBase.cpp:96
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::ObjectMaskFrameData::HasData
bool HasData() const
Definition: ObjectMask.h:34
openshot::ObjectMask::LoadObjMaskData
bool LoadObjMaskData(std::string inputFilePath)
Definition: ObjectMask.cpp:170
openshot::ObjectMaskData::height
int height
Definition: TrackedObjectBBox.h:30
openshot::ObjectMaskData::width
int width
Definition: TrackedObjectBBox.h:29
openshot::EffectBase::info
EffectInfoStruct info
Information about the current effect.
Definition: EffectBase.h:114
openshot::BBox::height
float height
bounding box height
Definition: TrackedObjectBBox.h:51
openshot::ObjectMaskFrameData::box
BBox box
Definition: ObjectMask.h:31
openshot::ObjectMask::ObjectMask
ObjectMask()
Definition: ObjectMask.cpp:100
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AnimatedCurve.h:24
openshot::EffectBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
Definition: EffectBase.cpp:676
openshot::ClipBase::add_property_choice_json
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
Definition: ClipBase.cpp:132
openshot::EffectBase::JsonValue
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: EffectBase.cpp:102
openshot::BBox::cy
float cy
y-coordinate of the bounding box center
Definition: TrackedObjectBBox.h:49
openshot::ObjectMaskFrameData::rle
std::vector< uint32_t > rle
Definition: ObjectMask.h:30
openshot::Keyframe::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: KeyFrame.cpp:372
openshot::EffectBase::trackedObjects
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Definition: EffectBase.h:111
openshot::ObjectMask::GetFrame
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
Definition: ObjectMask.cpp:122
openshot::Keyframe::JsonValue
Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: KeyFrame.cpp:339
openshot::ObjectMask::Json
std::string Json() const override
Generate JSON string of this object.
Definition: ObjectMask.cpp:247
openshot::ObjectMask::PropertiesJSON
std::string PropertiesJSON(int64_t requested_frame) const override
Definition: ObjectMask.cpp:304
ObjectMask.h
Header file for Object Mask effect class.
openshot::EffectBase::BasePropertiesJSON
Json::Value BasePropertiesJSON(int64_t requested_frame) const
Generate JSON object of base properties (recommended to be used by all effects)
Definition: EffectBase.cpp:257
openshot::ObjectMask::mask_alpha
Keyframe mask_alpha
Definition: ObjectMask.h:51
openshot::ObjectMaskData
Definition: TrackedObjectBBox.h:27
openshot::ObjectMask::draw_mask
Keyframe draw_mask
Definition: ObjectMask.h:49
openshot::Keyframe
A Keyframe is a collection of Point instances, which is used to vary a number or property over time.
Definition: KeyFrame.h:53
openshot::Color::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: Color.cpp:117
openshot::ObjectMask::mask_color
Color mask_color
Definition: ObjectMask.h:50
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:223
openshot::BBox::width
float width
bounding box width
Definition: TrackedObjectBBox.h:50
openshot::ObjectMask::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: ObjectMask.cpp:275
openshot::EffectBase::InitEffectInfo
void InitEffectInfo()
Definition: EffectBase.cpp:42
openshot::Color::green
openshot::Keyframe green
Curve representing the green value (0 - 255)
Definition: Color.h:31
openshot::ObjectMaskFrameData::score
float score
Definition: ObjectMask.h:32
openshot::EffectInfoStruct::has_audio
bool has_audio
Determines if this effect manipulates the audio of a frame.
Definition: EffectBase.h:44
openshot::ObjectMaskFrameData
Definition: ObjectMask.h:27
openshot::EffectInfoStruct::has_tracked_object
bool has_tracked_object
Determines if this effect track objects through the clip.
Definition: EffectBase.h:45
openshot::ObjectMask::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: ObjectMask.cpp:266
Frame.h
Header file for Frame class.
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:193
openshot::ObjectMask::stroke_width
Keyframe stroke_width
Definition: ObjectMask.h:54
openshot::EffectInfoStruct::class_name
std::string class_name
The class name of the effect.
Definition: EffectBase.h:39
openshot::Color::JsonValue
Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: Color.cpp:86
openshot::EffectInfoStruct::description
std::string description
The description of this effect and what it does.
Definition: EffectBase.h:41
openshot::BBox
This struct holds the information of a bounding-box.
Definition: TrackedObjectBBox.h:46
openshot::EffectInfoStruct::has_video
bool has_video
Determines if this effect manipulates the image of a frame.
Definition: EffectBase.h:43
openshot::ObjectMask::TrackedObjectMask
std::shared_ptr< QImage > TrackedObjectMask(std::shared_ptr< QImage > target_image, int64_t frame_number) const override
Generate a black/white mask from tracked object data.
Definition: ObjectMask.cpp:226
openshot::ClipBase::Id
void Id(std::string value)
Definition: ClipBase.h:94
openshot::ObjectMaskFrameData::width
int width
Definition: ObjectMask.h:28
openshot::Color::GetColorRGBA
std::vector< int > GetColorRGBA(int64_t frame_number)
Definition: Color.cpp:58
openshot::EffectInfoStruct::name
std::string name
The name of the effect.
Definition: EffectBase.h:40
openshot::ObjectMask::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: ObjectMask.cpp:252
openshot::ObjectMask::stroke_color
Color stroke_color
Definition: ObjectMask.h:52
openshot::ObjectMaskFrameData::height
int height
Definition: ObjectMask.h:29
openshot::BBox::cx
float cx
x-coordinate of the bounding box center
Definition: TrackedObjectBBox.h:48
openshot::Color::red
openshot::Keyframe red
Curve representing the red value (0 - 255)
Definition: Color.h:30
openshot::ObjectMaskData::rle
std::vector< uint32_t > rle
Definition: TrackedObjectBBox.h:31
openshot::Color::blue
openshot::Keyframe blue
Curve representing the red value (0 - 255)
Definition: Color.h:32
Exceptions.h
Header file for all Exception classes.
openshot::EffectBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: EffectBase.cpp:146
openshot::Keyframe::GetValue
double GetValue(int64_t index) const
Get the value at a specific index.
Definition: KeyFrame.cpp:258
openshot::ObjectMask::stroke_alpha
Keyframe stroke_alpha
Definition: ObjectMask.h:53
Generated by   doxygen 1.8.17