OpenShot Library | libopenshot 0.2.7
CVStabilization.cpp
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Source file for CVStabilization class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 * @author Brenno Caldato <brenno.caldato@outlook.com>
6 *
7 * @ref License
8 */
9
10/* LICENSE
11 *
12 * Copyright (c) 2008-2019 OpenShot Studios, LLC
13 * <http://www.openshotstudios.com/>. This file is part of
14 * OpenShot Library (libopenshot), an open-source project dedicated to
15 * delivering high quality video editing and animation solutions to the
16 * world. For more information visit <http://www.openshot.org/>.
17 *
18 * OpenShot Library (libopenshot) is free software: you can redistribute it
19 * and/or modify it under the terms of the GNU Lesser General Public License
20 * as published by the Free Software Foundation, either version 3 of the
21 * License, or (at your option) any later version.
22 *
23 * OpenShot Library (libopenshot) is distributed in the hope that it will be
24 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU Lesser General Public License for more details.
27 *
28 * You should have received a copy of the GNU Lesser General Public License
29 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30 */
31
32#include <fstream>
33#include <iomanip>
34#include <iostream>
35
36#include "CVStabilization.h"
37#include <google/protobuf/util/time_util.h>
38
39using namespace std;
40using namespace openshot;
41using google::protobuf::util::TimeUtil;
42
43// Set default smoothing window value to compute stabilization
44CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController)
45: processingController(&processingController){
46 SetJson(processInfoJson);
47 start = 1;
48 end = 1;
49}
50
51// Process clip and store necessary stabilization data
52void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
53
54 if(error){
55 return;
56 }
57 processingController->SetError(false, "");
58
59 start = _start; end = _end;
60 // Compute max and average transformation parameters
61 avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0;
62
63 video.Open();
64 // Save original video width and height
65 cv::Size readerDims(video.Reader()->info.width, video.Reader()->info.height);
66
67 size_t frame_number;
68 if(!process_interval || end <= 1 || end-start == 0){
69 // Get total number of frames in video
70 start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
71 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
72 }
73
74 // Extract and track opticalflow features for each frame
75 for (frame_number = start; frame_number <= end; frame_number++)
76 {
77 // Stop the feature tracker process
78 if(processingController->ShouldStop()){
79 return;
80 }
81
82 std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
83
84 // Grab OpenCV Mat image
85 cv::Mat cvimage = f->GetImageCV();
86 // Resize frame to original video width and height if they differ
87 if(cvimage.size().width != readerDims.width || cvimage.size().height != readerDims.height)
88 cv::resize(cvimage, cvimage, cv::Size(readerDims.width, readerDims.height));
89 cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
90
91 if(!TrackFrameFeatures(cvimage, frame_number)){
92 prev_to_cur_transform.push_back(TransformParam(0, 0, 0));
93 }
94
95 // Update progress
96 processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
97 }
98
99 // Calculate trajectory data
100 std::vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
101
102 // Calculate and save smoothed trajectory data
103 trajectoryData = SmoothTrajectory(trajectory);
104
105 // Calculate and save transformation data
106 transformationData = GenNewCamPosition(trajectoryData);
107
108 // Normalize smoothed trajectory data
109 for(auto &dataToNormalize : trajectoryData){
110 dataToNormalize.second.x/=readerDims.width;
111 dataToNormalize.second.y/=readerDims.height;
112 }
113 // Normalize transformation data
114 for(auto &dataToNormalize : transformationData){
115 dataToNormalize.second.dx/=readerDims.width;
116 dataToNormalize.second.dy/=readerDims.height;
117 }
118}
119
120// Track current frame features and find the relative transformation
121bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
122 // Check if there are black frames
123 if(cv::countNonZero(frame) < 1){
124 return false;
125 }
126
127 // Initialize prev_grey if not
128 if(prev_grey.empty()){
129 prev_grey = frame;
130 return true;
131 }
132
133 // OpticalFlow features vector
134 std::vector <cv::Point2f> prev_corner, cur_corner;
135 std::vector <cv::Point2f> prev_corner2, cur_corner2;
136 std::vector <uchar> status;
137 std::vector <float> err;
138 // Extract new image features
139 cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
140 // Track features
141 cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err);
142 // Remove untracked features
143 for(size_t i=0; i < status.size(); i++) {
144 if(status[i]) {
145 prev_corner2.push_back(prev_corner[i]);
146 cur_corner2.push_back(cur_corner[i]);
147 }
148 }
149 // In case no feature was detected
150 if(prev_corner2.empty() || cur_corner2.empty()){
151 last_T = cv::Mat();
152 // prev_grey = cv::Mat();
153 return false;
154 }
155
156 // Translation + rotation only
157 cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
158
159 double da, dx, dy;
160 // If T has nothing inside return (probably a segment where there is nothing to stabilize)
161 if(T.size().width == 0 || T.size().height == 0){
162 return false;
163 }
164 else{
165 // If no transformation is found, just use the last known good transform
166 if(T.data == NULL){
167 if(!last_T.empty())
168 last_T.copyTo(T);
169 else
170 return false;
171 }
172 // Decompose T
173 dx = T.at<double>(0,2);
174 dy = T.at<double>(1,2);
175 da = atan2(T.at<double>(1,0), T.at<double>(0,0));
176 }
177
178 // Filter transformations parameters, if they are higher than these: return
179 if(dx > 200 || dy > 200 || da > 0.1){
180 return false;
181 }
182
183 // Keep computing average and max transformation parameters
184 avr_dx+=fabs(dx);
185 avr_dy+=fabs(dy);
186 avr_da+=fabs(da);
187 if(fabs(dx) > max_dx)
188 max_dx = dx;
189 if(fabs(dy) > max_dy)
190 max_dy = dy;
191 if(fabs(da) > max_da)
192 max_da = da;
193
194 T.copyTo(last_T);
195
196 prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
197 frame.copyTo(prev_grey);
198
199 return true;
200}
201
202std::vector<CamTrajectory> CVStabilization::ComputeFramesTrajectory(){
203
204 // Accumulated frame to frame transform
205 double a = 0;
206 double x = 0;
207 double y = 0;
208
209 vector <CamTrajectory> trajectory; // trajectory at all frames
210
211 // Compute global camera trajectory. First frame is the origin
212 for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
213 x += prev_to_cur_transform[i].dx;
214 y += prev_to_cur_transform[i].dy;
215 a += prev_to_cur_transform[i].da;
216
217 // Save trajectory data to vector
218 trajectory.push_back(CamTrajectory(x,y,a));
219 }
220 return trajectory;
221}
222
223std::map<size_t,CamTrajectory> CVStabilization::SmoothTrajectory(std::vector <CamTrajectory> &trajectory){
224
225 std::map <size_t,CamTrajectory> smoothed_trajectory; // trajectory at all frames
226
227 for(size_t i=0; i < trajectory.size(); i++) {
228 double sum_x = 0;
229 double sum_y = 0;
230 double sum_a = 0;
231 int count = 0;
232
233 for(int j=-smoothingWindow; j <= smoothingWindow; j++) {
234 if(i+j < trajectory.size()) {
235 sum_x += trajectory[i+j].x;
236 sum_y += trajectory[i+j].y;
237 sum_a += trajectory[i+j].a;
238
239 count++;
240 }
241 }
242
243 double avg_a = sum_a / count;
244 double avg_x = sum_x / count;
245 double avg_y = sum_y / count;
246
247 // Add smoothed trajectory data to map
248 smoothed_trajectory[i + start] = CamTrajectory(avg_x, avg_y, avg_a);
249 }
250 return smoothed_trajectory;
251}
252
253// Generate new transformations parameters for each frame to follow the smoothed trajectory
254std::map<size_t,TransformParam> CVStabilization::GenNewCamPosition(std::map <size_t,CamTrajectory> &smoothed_trajectory){
255 std::map <size_t,TransformParam> new_prev_to_cur_transform;
256
257 // Accumulated frame to frame transform
258 double a = 0;
259 double x = 0;
260 double y = 0;
261
262 for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
263 x += prev_to_cur_transform[i].dx;
264 y += prev_to_cur_transform[i].dy;
265 a += prev_to_cur_transform[i].da;
266
267 // target - current
268 double diff_x = smoothed_trajectory[i + start].x - x;
269 double diff_y = smoothed_trajectory[i + start].y - y;
270 double diff_a = smoothed_trajectory[i + start].a - a;
271
272 double dx = prev_to_cur_transform[i].dx + diff_x;
273 double dy = prev_to_cur_transform[i].dy + diff_y;
274 double da = prev_to_cur_transform[i].da + diff_a;
275
276 // Add transformation data to map
277 new_prev_to_cur_transform[i + start] = TransformParam(dx, dy, da);
278 }
279 return new_prev_to_cur_transform;
280}
281
282// Save stabilization data to protobuf file
284 using std::ios;
285
286 // Create stabilization message
287 pb_stabilize::Stabilization stabilizationMessage;
288
289 std::map<size_t,CamTrajectory>::iterator trajData = trajectoryData.begin();
290 std::map<size_t,TransformParam>::iterator transData = transformationData.begin();
291
292 // Iterate over all frames data and save in protobuf message
293 for(; trajData != trajectoryData.end(); ++trajData, ++transData){
294 AddFrameDataToProto(stabilizationMessage.add_frame(), trajData->second, transData->second, trajData->first);
295 }
296 // Add timestamp
297 *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
298
299 // Write the new message to disk.
300 std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
301 if (!stabilizationMessage.SerializeToOstream(&output)) {
302 std::cerr << "Failed to write protobuf message." << std::endl;
303 return false;
304 }
305
306 // Delete all global objects allocated by libprotobuf.
307 google::protobuf::ShutdownProtobufLibrary();
308
309 return true;
310}
311
312// Add frame stabilization data into protobuf message
313void CVStabilization::AddFrameDataToProto(pb_stabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number){
314
315 // Save frame number
316 pbFrameData->set_id(frame_number);
317
318 // Save camera trajectory data
319 pbFrameData->set_a(trajData.a);
320 pbFrameData->set_x(trajData.x);
321 pbFrameData->set_y(trajData.y);
322
323 // Save transformation data
324 pbFrameData->set_da(transData.da);
325 pbFrameData->set_dx(transData.dx);
326 pbFrameData->set_dy(transData.dy);
327}
328
330
331 // Check if the stabilizer info for the requested frame exists
332 if ( transformationData.find(frameId) == transformationData.end() ) {
333
334 return TransformParam();
335 } else {
336
337 return transformationData[frameId];
338 }
339}
340
342
343 // Check if the stabilizer info for the requested frame exists
344 if ( trajectoryData.find(frameId) == trajectoryData.end() ) {
345
346 return CamTrajectory();
347 } else {
348
349 return trajectoryData[frameId];
350 }
351}
352
353// Load JSON string into this object
354void CVStabilization::SetJson(const std::string value) {
355 // Parse JSON string into JSON objects
356 try
357 {
358 const Json::Value root = openshot::stringToJson(value);
359 // Set all values that match
360
361 SetJsonValue(root);
362 }
363 catch (const std::exception& e)
364 {
365 // Error parsing JSON (or missing keys)
366 throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
367 }
368}
369
370// Load Json::Value into this object
371void CVStabilization::SetJsonValue(const Json::Value root) {
372
373 // Set data from Json (if key is found)
374 if (!root["protobuf_data_path"].isNull()){
375 protobuf_data_path = (root["protobuf_data_path"].asString());
376 }
377 if (!root["smoothing-window"].isNull()){
378 smoothingWindow = (root["smoothing-window"].asInt());
379 }
380}
381
382/*
383||||||||||||||||||||||||||||||||||||||||||||||||||
384 ONLY FOR MAKE TEST
385||||||||||||||||||||||||||||||||||||||||||||||||||
386*/
387
388// Load protobuf data file
390 using std::ios;
391 // Create stabilization message
392 pb_stabilize::Stabilization stabilizationMessage;
393 // Read the existing tracker message.
394 std::fstream input(protobuf_data_path, ios::in | ios::binary);
395 if (!stabilizationMessage.ParseFromIstream(&input)) {
396 std::cerr << "Failed to parse protobuf message." << std::endl;
397 return false;
398 }
399
400 // Make sure the data maps are empty
401 transformationData.clear();
402 trajectoryData.clear();
403
404 // Iterate over all frames of the saved message and assign to the data maps
405 for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) {
406 const pb_stabilize::Frame& pbFrameData = stabilizationMessage.frame(i);
407
408 // Load frame number
409 size_t id = pbFrameData.id();
410
411 // Load camera trajectory data
412 float x = pbFrameData.x();
413 float y = pbFrameData.y();
414 float a = pbFrameData.a();
415
416 // Assign data to trajectory map
417 trajectoryData[id] = CamTrajectory(x,y,a);
418
419 // Load transformation data
420 float dx = pbFrameData.dx();
421 float dy = pbFrameData.dy();
422 float da = pbFrameData.da();
423
424 // Assing data to transformation map
425 transformationData[id] = TransformParam(dx,dy,da);
426 }
427
428 // Delete all global objects allocated by libprotobuf.
429 google::protobuf::ShutdownProtobufLibrary();
430
431 return true;
432}
Header file for CVStabilization class.
TransformParam GetTransformParamData(size_t frameId)
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
CVStabilization(std::string processInfoJson, ProcessingController &processingController)
Set default smoothing window value to compute stabilization.
void AddFrameDataToProto(pb_stabilize::Frame *pbFrameData, CamTrajectory &trajData, TransformParam &transData, size_t frame_number)
Add frame stabilization data into protobuf message.
std::map< size_t, TransformParam > transformationData
void stabilizeClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Process clip and store necessary stabilization data.
std::map< size_t, CamTrajectory > trajectoryData
CamTrajectory GetCamTrajectoryTrackedData(size_t frameId)
void SetJson(const std::string value)
Load JSON string into this object.
void SetError(bool err, std::string message)
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
void Open() override
Open the internal reader.
Definition: Clip.cpp:302
float End() const
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve.
Definition: Clip.cpp:338
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition: Clip.cpp:279
Exception for invalid JSON.
Definition: Exceptions.h:206
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34