OpenShot Library | libopenshot 0.2.7
Frame.cpp
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Source file for Frame class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9/* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31#include "Frame.h"
32#include <OpenShotAudio.h>
33
34#include <QApplication>
35#include <QImage>
36#include <QPixmap>
37#include <QBitmap>
38#include <QColor>
39#include <QString>
40#include <QVector>
41#include <QPainter>
42#include <QHBoxLayout>
43#include <QWidget>
44#include <QLabel>
45#include <QPointF>
46#include <QWidget>
47
48#include <thread> // for std::this_thread::sleep_for
49#include <chrono> // for std::chrono::milliseconds
50
51using namespace std;
52using namespace openshot;
53
54// Constructor - image & audio
55Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
56 : audio(std::make_shared<juce::AudioSampleBuffer>(channels, samples)),
57 number(number), width(width), height(height),
58 pixel_ratio(1,1), color(color), qbuffer(NULL),
59 channels(channels), channel_layout(LAYOUT_STEREO),
60 sample_rate(44100),
61 has_audio_data(false), has_image_data(false),
62 max_audio_sample(0)
63{
64 // zero (fill with silence) the audio buffer
65 audio->clear();
66}
67
68// Delegating Constructor - blank frame
69Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {};
70
71// Delegating Constructor - image only
72Frame::Frame(int64_t number, int width, int height, std::string color)
73 : Frame::Frame(number, width, height, color, 0, 2) {};
74
75// Delegating Constructor - audio only
76Frame::Frame(int64_t number, int samples, int channels)
77 : Frame::Frame(number, 1, 1, "#000000", samples, channels) {};
78
79
80// Copy constructor
81Frame::Frame ( const Frame &other )
82{
83 // copy pointers and data
84 DeepCopy(other);
85}
86
87// Assignment operator
89{
90 // copy pointers and data
91 DeepCopy(other);
92
93 return *this;
94}
95
96// Copy data and pointers from another Frame instance
97void Frame::DeepCopy(const Frame& other)
98{
99 number = other.number;
100 channels = other.channels;
101 width = other.width;
102 height = other.height;
103 channel_layout = other.channel_layout;
106 sample_rate = other.sample_rate;
107 pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
108 color = other.color;
109 max_audio_sample = other.max_audio_sample;
110
111 if (other.image)
112 image = std::make_shared<QImage>(*(other.image));
113 if (other.audio)
114 audio = std::make_shared<juce::AudioSampleBuffer>(*(other.audio));
115 if (other.wave_image)
116 wave_image = std::make_shared<QImage>(*(other.wave_image));
117}
118
119// Destructor
121 // Clear all pointers
122 image.reset();
123 audio.reset();
124 #ifdef USE_OPENCV
125 imagecv.release();
126 #endif
127}
128
129// Display the frame image to the screen (primarily used for debugging reasons)
131{
132 if (!QApplication::instance()) {
133 // Only create the QApplication once
134 static int argc = 1;
135 static char* argv[1] = {NULL};
136 previewApp = std::make_shared<QApplication>(argc, argv);
137 }
138
139 // Get preview image
140 std::shared_ptr<QImage> previewImage = GetImage();
141
142 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
143 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
144 {
145 // Calculate correct DAR (display aspect ratio)
146 int new_width = previewImage->size().width();
147 int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
148
149 // Resize to fix DAR
150 previewImage = std::make_shared<QImage>(previewImage->scaled(
151 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
152 }
153
154 // Create window
155 QWidget previewWindow;
156 previewWindow.setStyleSheet("background-color: #000000;");
157 QHBoxLayout layout;
158
159 // Create label with current frame's image
160 QLabel previewLabel;
161 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163 layout.addWidget(&previewLabel);
164
165 // Show the window
166 previewWindow.setLayout(&layout);
167 previewWindow.show();
168 previewApp->exec();
169}
170
171// Get an audio waveform image
172std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
173{
174 // Clear any existing waveform image
176
177 // Init a list of lines
178 QVector<QPointF> lines;
179 QVector<QPointF> labels;
180
181 // Calculate width of an image based on the # of samples
182 int total_samples = GetAudioSamplesCount();
183 if (total_samples > 0)
184 {
185 // If samples are present...
186 int new_height = 200 * audio->getNumChannels();
187 int height_padding = 20 * (audio->getNumChannels() - 1);
188 int total_height = new_height + height_padding;
189 int total_width = 0;
190
191 // Loop through each audio channel
192 float Y = 100.0;
193 for (int channel = 0; channel < audio->getNumChannels(); channel++)
194 {
195 float X = 0.0;
196
197 // Get audio for this channel
198 const float *samples = audio->getReadPointer(channel);
199
200 for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
201 {
202 // Sample value (scaled to -100 to 100)
203 float value = samples[sample] * 100.0;
204
205 // Append a line segment for each sample
206 lines.push_back(QPointF(X,Y+1.0));
207 lines.push_back(QPointF(X,(Y-value)+1.0));
208 }
209
210 // Add Channel Label Coordinate
211 labels.push_back(QPointF(5.0, Y - 5.0));
212
213 // Increment Y
214 Y += (200 + height_padding);
215 total_width = X;
216 }
217
218 // Create blank image
219 wave_image = std::make_shared<QImage>(
220 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
221 wave_image->fill(QColor(0,0,0,0));
222
223 // Load QPainter with wave_image device
224 QPainter painter(wave_image.get());
225
226 // Set pen color
227 QPen pen;
228 pen.setColor(QColor(Red, Green, Blue, Alpha));
229 pen.setWidthF(1.0);
230 pen.setStyle(Qt::SolidLine);
231 painter.setPen(pen);
232
233 // Draw the waveform
234 painter.drawLines(lines);
235 painter.end();
236
237 // Resize Image (if requested)
238 if (width != total_width || height != total_height) {
239 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
240 wave_image = std::make_shared<QImage>(scaled_wave_image);
241 }
242 }
243 else
244 {
245 // No audio samples present
246 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
247 wave_image->fill(QColor(QString::fromStdString("#000000")));
248 }
249
250 // Return new image
251 return wave_image;
252}
253
254// Clear the waveform image (and deallocate its memory)
256{
257 if (wave_image)
258 wave_image.reset();
259}
260
261// Get an audio waveform image pixels
262const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
263{
264 // Get audio wave form image
265 wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
266
267 // Return array of pixel packets
268 return wave_image->constBits();
269}
270
271// Display the wave form
273{
274 // Get audio wave form image
275 GetWaveform(720, 480, 0, 123, 255, 255);
276
277 if (!QApplication::instance()) {
278 // Only create the QApplication once
279 static int argc = 1;
280 static char* argv[1] = {NULL};
281 previewApp = std::make_shared<QApplication>(argc, argv);
282 }
283
284 // Create window
285 QWidget previewWindow;
286 previewWindow.setStyleSheet("background-color: #000000;");
287 QHBoxLayout layout;
288
289 // Create label with current frame's waveform image
290 QLabel previewLabel;
291 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
292 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
293 layout.addWidget(&previewLabel);
294
295 // Show the window
296 previewWindow.setLayout(&layout);
297 previewWindow.show();
298 previewApp->exec();
299
300 // Deallocate waveform image
302}
303
304// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
305float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
306{
307 if (channel > 0) {
308 // return average magnitude for a specific channel/sample range
309 return audio->getMagnitude(channel, sample, magnitude_range);
310
311 } else {
312 // Return average magnitude for all channels
313 return audio->getMagnitude(sample, magnitude_range);
314 }
315}
316
317// Get an array of sample data
318float* Frame::GetAudioSamples(int channel)
319{
320 // return JUCE audio data for this channel
321 return audio->getWritePointer(channel);
322}
323
324// Get a planar array of sample data, using any sample rate
325float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
326{
327 float *output = NULL;
328 juce::AudioSampleBuffer *buffer(audio.get());
329 int num_of_channels = audio->getNumChannels();
330 int num_of_samples = GetAudioSamplesCount();
331
332 // Resample to new sample rate (if needed)
333 if (new_sample_rate != sample_rate)
334 {
335 // YES, RESAMPLE AUDIO
336 resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
337
338 // Resample data, and return new buffer pointer
339 buffer = resampler->GetResampledBuffer();
340
341 // Update num_of_samples
342 num_of_samples = buffer->getNumSamples();
343 }
344
345 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
346 output = new float[num_of_channels * num_of_samples];
347 int position = 0;
348
349 // Loop through samples in each channel (combining them)
350 for (int channel = 0; channel < num_of_channels; channel++)
351 {
352 for (int sample = 0; sample < num_of_samples; sample++)
353 {
354 // Add sample to output array
355 output[position] = buffer->getReadPointer(channel)[sample];
356
357 // increment position
358 position++;
359 }
360 }
361
362 // Update sample count (since it might have changed due to resampling)
363 *sample_count = num_of_samples;
364
365 // return combined array
366 return output;
367}
368
369
370// Get an array of sample data (all channels interleaved together), using any sample rate
371float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
372{
373 float *output = NULL;
374 juce::AudioSampleBuffer *buffer(audio.get());
375 int num_of_channels = audio->getNumChannels();
376 int num_of_samples = GetAudioSamplesCount();
377
378 // Resample to new sample rate (if needed)
379 if (new_sample_rate != sample_rate && resampler)
380 {
381 // YES, RESAMPLE AUDIO
382 resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
383
384 // Resample data, and return new buffer pointer
385 buffer = resampler->GetResampledBuffer();
386
387 // Update num_of_samples
388 num_of_samples = buffer->getNumSamples();
389 }
390
391 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
392 output = new float[num_of_channels * num_of_samples];
393 int position = 0;
394
395 // Loop through samples in each channel (combining them)
396 for (int sample = 0; sample < num_of_samples; sample++)
397 {
398 for (int channel = 0; channel < num_of_channels; channel++)
399 {
400 // Add sample to output array
401 output[position] = buffer->getReadPointer(channel)[sample];
402
403 // increment position
404 position++;
405 }
406 }
407
408 // Update sample count (since it might have changed due to resampling)
409 *sample_count = num_of_samples;
410
411 // return combined array
412 return output;
413}
414
415// Get number of audio channels
417{
418 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
419 if (audio)
420 return audio->getNumChannels();
421 else
422 return 0;
423}
424
425// Get number of audio samples
427{
428 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
429 return max_audio_sample;
430}
431
432juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
433{
434 return audio.get();
435}
436
437// Get the size in bytes of this frame (rough estimate)
439{
440 int64_t total_bytes = 0;
441 if (image)
442 total_bytes += (width * height * sizeof(char) * 4);
443 if (audio) {
444 // approximate audio size (sample rate / 24 fps)
445 total_bytes += (sample_rate / 24.0) * sizeof(float);
446 }
447
448 // return size of this frame
449 return total_bytes;
450}
451
452// Get pixel data (as packets)
453const unsigned char* Frame::GetPixels()
454{
455 // Check for blank image
456 if (!image)
457 // Fill with black
458 AddColor(width, height, color);
459
460 // Return array of pixel packets
461 return image->constBits();
462}
463
464// Get pixel data (for only a single scan-line)
465const unsigned char* Frame::GetPixels(int row)
466{
467 // Check for blank image
468 if (!image)
469 // Fill with black
470 AddColor(width, height, color);
471
472 // Return array of pixel packets
473 return image->constScanLine(row);
474}
475
476// Check a specific pixel color value (returns True/False)
477bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
478 int col_pos = col * 4; // Find column array position
479 if (!image || row < 0 || row >= (height - 1) ||
480 col_pos < 0 || col_pos >= (width - 1) ) {
481 // invalid row / col
482 return false;
483 }
484 // Check pixel color
485 const unsigned char* pixels = GetPixels(row);
486 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
487 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
488 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
489 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
490 // Pixel color matches successfully
491 return true;
492 } else {
493 // Pixel color does not match
494 return false;
495 }
496}
497
498// Set Pixel Aspect Ratio
499void Frame::SetPixelRatio(int num, int den)
500{
501 pixel_ratio.num = num;
502 pixel_ratio.den = den;
503}
504
505// Set frame number
506void Frame::SetFrameNumber(int64_t new_number)
507{
508 number = new_number;
509}
510
511// Calculate the # of samples per video frame (for a specific frame number and frame rate)
512int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
513{
514 // Get the total # of samples for the previous frame, and the current frame (rounded)
515 double fps_rate = fps.Reciprocal().ToDouble();
516
517 // Determine previous samples total, and make sure it's evenly divisible by the # of channels
518 double previous_samples = (sample_rate * fps_rate) * (number - 1);
519 double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
520 previous_samples -= previous_samples_remainder;
521
522 // Determine the current samples total, and make sure it's evenly divisible by the # of channels
523 double total_samples = (sample_rate * fps_rate) * number;
524 double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
525 total_samples -= total_samples_remainder;
526
527 // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
528 // be evenly divided into frames, so each frame can have have different # of samples.
529 int samples_per_frame = round(total_samples - previous_samples);
530 if (samples_per_frame < 0)
531 samples_per_frame = 0;
532 return samples_per_frame;
533}
534
535// Calculate the # of samples per video frame (for the current frame number)
536int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
537{
538 return GetSamplesPerFrame(number, fps, sample_rate, channels);
539}
540
541// Get height of image
543{
544 return height;
545}
546
547// Get height of image
549{
550 return width;
551}
552
553// Get the original sample rate of this frame's audio data
555{
556 return sample_rate;
557}
558
559// Get the original sample rate of this frame's audio data
561{
562 return channel_layout;
563}
564
565
566// Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
567void Frame::Save(std::string path, float scale, std::string format, int quality)
568{
569 // Get preview image
570 std::shared_ptr<QImage> previewImage = GetImage();
571
572 // scale image if needed
573 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
574 {
575 int new_width = width;
576 int new_height = height;
577
578 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
579 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
580 {
581 // Calculate correct DAR (display aspect ratio)
582 int new_width = previewImage->size().width();
583 int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
584
585 // Resize to fix DAR
586 previewImage = std::make_shared<QImage>(previewImage->scaled(
587 new_width, new_height,
588 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
589 }
590
591 // Resize image
592 previewImage = std::make_shared<QImage>(previewImage->scaled(
593 new_width * scale, new_height * scale,
594 Qt::KeepAspectRatio, Qt::SmoothTransformation));
595 }
596
597 // Save image
598 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
599}
600
601// Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
602void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
603 std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
604
605 // Create blank thumbnail image & fill background color
606 auto thumbnail = std::make_shared<QImage>(
607 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
608 thumbnail->fill(QColor(QString::fromStdString(background_color)));
609
610 // Create painter
611 QPainter painter(thumbnail.get());
612 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
613
614 // Get preview image
615 std::shared_ptr<QImage> previewImage = GetImage();
616
617 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
618 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
619 {
620 // Calculate correct DAR (display aspect ratio)
621 int aspect_width = previewImage->size().width();
622 int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
623
624 // Resize to fix DAR
625 previewImage = std::make_shared<QImage>(previewImage->scaled(
626 aspect_width, aspect_height,
627 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
628 }
629
630 // Resize frame image
631 if (ignore_aspect)
632 // Ignore aspect ratio
633 previewImage = std::make_shared<QImage>(previewImage->scaled(
634 new_width, new_height,
635 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
636 else
637 // Maintain aspect ratio
638 previewImage = std::make_shared<QImage>(previewImage->scaled(
639 new_width, new_height,
640 Qt::KeepAspectRatio, Qt::SmoothTransformation));
641
642 // Composite frame image onto background (centered)
643 int x = (new_width - previewImage->size().width()) / 2.0; // center
644 int y = (new_height - previewImage->size().height()) / 2.0; // center
645 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
646
647
648 // Create transform and rotate (if needed)
649 QTransform transform;
650 float origin_x = previewImage->width() / 2.0;
651 float origin_y = previewImage->height() / 2.0;
652 transform.translate(origin_x, origin_y);
653 transform.rotate(rotate);
654 transform.translate(-origin_x,-origin_y);
655 painter.setTransform(transform);
656
657 // Draw image onto QImage
658 painter.drawImage(x, y, *previewImage);
659
660
661 // Overlay Image (if any)
662 if (overlay_path != "") {
663 // Open overlay
664 auto overlay = std::make_shared<QImage>();
665 overlay->load(QString::fromStdString(overlay_path));
666
667 // Set pixel format
668 overlay = std::make_shared<QImage>(
669 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
670
671 // Resize to fit
672 overlay = std::make_shared<QImage>(overlay->scaled(
673 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
674
675 // Composite onto thumbnail
676 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
677 painter.drawImage(0, 0, *overlay);
678 }
679
680
681 // Mask Image (if any)
682 if (mask_path != "") {
683 // Open mask
684 auto mask = std::make_shared<QImage>();
685 mask->load(QString::fromStdString(mask_path));
686
687 // Set pixel format
688 mask = std::make_shared<QImage>(
689 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
690
691 // Resize to fit
692 mask = std::make_shared<QImage>(mask->scaled(
693 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
694
695 // Negate mask
696 mask->invertPixels();
697
698 // Get pixels
699 unsigned char *pixels = (unsigned char *) thumbnail->bits();
700 const unsigned char *mask_pixels = (const unsigned char *) mask->constBits();
701
702 // Convert the mask image to grayscale
703 // Loop through pixels
704 for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
705 {
706 // Get the RGB values from the pixel
707 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
708 int Frame_Alpha = pixels[byte_index + 3];
709 int Mask_Value = constrain(Frame_Alpha - gray_value);
710
711 // Set all alpha pixels to gray value
712 pixels[byte_index + 3] = Mask_Value;
713 }
714 }
715
716
717 // End painter
718 painter.end();
719
720 // Save image
721 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
722}
723
724// Constrain a color value from 0 to 255
725int Frame::constrain(int color_value)
726{
727 // Constrain new color from 0 to 255
728 if (color_value < 0)
729 color_value = 0;
730 else if (color_value > 255)
731 color_value = 255;
732
733 return color_value;
734}
735
736// Add (or replace) pixel data to the frame (based on a solid color)
737void Frame::AddColor(int new_width, int new_height, std::string new_color)
738{
739 // Set color
740 color = new_color;
741
742 // Create new image object, and fill with pixel data
743 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
744 image = std::make_shared<QImage>(new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
745
746 // Fill with solid color
747 image->fill(QColor(QString::fromStdString(color)));
748
749 // Update height and width
750 width = image->width();
751 height = image->height();
752 has_image_data = true;
753}
754
755// Add (or replace) pixel data to the frame
757 int new_width, int new_height, int bytes_per_pixel,
758 QImage::Format type, const unsigned char *pixels_)
759{
760 // Create new buffer
761 {
762 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
763 qbuffer = pixels_;
764 } // Release addingImageSection lock
765
766 // Create new image object from pixel data
767 auto new_image = std::make_shared<QImage>(
768 qbuffer,
769 new_width, new_height,
770 new_width * bytes_per_pixel,
771 type,
772 (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer,
773 (void*) qbuffer
774 );
775 AddImage(new_image);
776}
777
778// Add (or replace) pixel data to the frame
779void Frame::AddImage(std::shared_ptr<QImage> new_image)
780{
781 // Ignore blank images
782 if (!new_image)
783 return;
784
785 // assign image data
786 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
787 image = new_image;
788
789 // Always convert to Format_RGBA8888_Premultiplied (if different)
790 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
791 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
792
793 // Update height and width
794 width = image->width();
795 height = image->height();
796 has_image_data = true;
797}
798
799// Add (or replace) pixel data to the frame (for only the odd or even lines)
800void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
801{
802 // Ignore blank new_image
803 if (!new_image)
804 return;
805
806 // Check for blank source image
807 if (!image) {
808 // Replace the blank source image
809 AddImage(new_image);
810
811 } else {
812 // Ignore image of different sizes or formats
813 bool ret=false;
814 if (image == new_image || image->size() != new_image->size()) {
815 ret = true;
816 }
817 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
818 new_image = std::make_shared<QImage>(
819 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
820 }
821 if (ret) {
822 return;
823 }
824
825 // Get the frame's image
826 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
827 unsigned char *pixels = image->bits();
828 const unsigned char *new_pixels = new_image->constBits();
829
830 // Loop through the scanlines of the image (even or odd)
831 int start = 0;
832 if (only_odd_lines)
833 start = 1;
834
835 for (int row = start; row < image->height(); row += 2) {
836 int offset = row * image->bytesPerLine();
837 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
838 }
839
840 // Update height and width
841 height = image->height();
842 width = image->width();
843 has_image_data = true;
844 }
845}
846
847
848// Resize audio container to hold more (or less) samples and channels
849void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
850{
851 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
852
853 // Resize JUCE audio buffer
854 audio->setSize(channels, length, true, true, false);
855 channel_layout = layout;
856 sample_rate = rate;
857
858 // Calculate max audio sample added
859 max_audio_sample = length;
860}
861
862// Add audio samples to a specific channel
863void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
864 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
865
866 // Clamp starting sample to 0
867 int destStartSampleAdjusted = max(destStartSample, 0);
868
869 // Extend audio container to hold more (or less) samples and channels.. if needed
870 int new_length = destStartSampleAdjusted + numSamples;
871 int new_channel_length = audio->getNumChannels();
872 if (destChannel >= new_channel_length)
873 new_channel_length = destChannel + 1;
874 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
875 audio->setSize(new_channel_length, new_length, true, true, false);
876
877 // Clear the range of samples first (if needed)
878 if (replaceSamples)
879 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
880
881 // Add samples to frame's audio buffer
882 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
883 has_audio_data = true;
884
885 // Calculate max audio sample added
886 if (new_length > max_audio_sample)
887 max_audio_sample = new_length;
888}
889
890// Apply gain ramp (i.e. fading volume)
891void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
892{
893 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
894
895 // Apply gain ramp
896 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
897}
898
899// Get pointer to Magick++ image object
900std::shared_ptr<QImage> Frame::GetImage()
901{
902 // Check for blank image
903 if (!image)
904 // Fill with black
905 AddColor(width, height, color);
906
907 return image;
908}
909
910#ifdef USE_OPENCV
911
912// Convert Qimage to Mat
913cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
914
915 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
916 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
917 int from_to[] = { 0,0, 1,1, 2,2 };
918 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
919 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
920 return mat2;
921};
922
923// Get pointer to OpenCV image object
925{
926 // Check for blank image
927 if (!image)
928 // Fill with black
929 AddColor(width, height, color);
930
931 // if (imagecv.empty())
932 // Convert Qimage to Mat
933 imagecv = Qimage2mat(image);
934
935 return imagecv;
936}
937
938std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
939 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
940 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
941
942 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
943
944 // Always convert to RGBA8888 (if different)
945 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
946 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
947
948 return imgIn;
949}
950
951// Set pointer to OpenCV image object
952void Frame::SetImageCV(cv::Mat _image)
953{
954 imagecv = _image;
955 image = Mat2Qimage(_image);
956}
957#endif
958
959#ifdef USE_IMAGEMAGICK
960// Get pointer to ImageMagick image object
961std::shared_ptr<Magick::Image> Frame::GetMagickImage()
962{
963 // Check for blank image
964 if (!image)
965 // Fill with black
966 AddColor(width, height, "#000000");
967
968 // Get the pixels from the frame image
969 const QRgb *tmpBits = (const QRgb*)image->constBits();
970
971 // Create new image object, and fill with pixel data
972 auto magick_image = std::make_shared<Magick::Image>(
973 image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits);
974
975 // Give image a transparent background color
976 magick_image->backgroundColor(Magick::Color("none"));
977 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
978 MAGICK_IMAGE_ALPHA(magick_image, true);
979
980 return magick_image;
981}
982#endif
983
984#ifdef USE_IMAGEMAGICK
985// Get pointer to QImage of frame
986void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
987{
988 const int BPP = 4;
989 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
990
991 /// Use realloc for fast memory allocation.
992 /// TODO: consider locking the buffer for mt safety
993 //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
994 qbuffer = new unsigned char[bufferSize]();
995 unsigned char *buffer = (unsigned char*)qbuffer;
996
997 MagickCore::ExceptionInfo exception;
998 // TODO: Actually do something, if we get an exception here
999 MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(), "RGBA", Magick::CharPixel, buffer, &exception);
1000
1001 // Create QImage of frame data
1002 image = std::make_shared<QImage>(
1003 qbuffer, width, height, width * BPP, QImage::Format_RGBA8888_Premultiplied,
1004 (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer);
1005
1006 // Update height and width
1007 width = image->width();
1008 height = image->height();
1009 has_image_data = true;
1010}
1011#endif
1012
1013// Play audio samples for this frame
1015{
1016 // Check if samples are present
1017 if (!GetAudioSamplesCount())
1018 return;
1019
1020 juce::AudioDeviceManager deviceManager;
1021 juce::String error = deviceManager.initialise (
1022 0, /* number of input channels */
1023 2, /* number of output channels */
1024 0, /* no XML settings.. */
1025 true /* select default device on failure */);
1026
1027 // Output error (if any)
1028 if (error.isNotEmpty()) {
1029 cout << "Error on initialise(): " << error << endl;
1030 }
1031
1032 juce::AudioSourcePlayer audioSourcePlayer;
1033 deviceManager.addAudioCallback (&audioSourcePlayer);
1034
1035 std::unique_ptr<AudioBufferSource> my_source;
1036 my_source.reset (new AudioBufferSource (audio.get()));
1037
1038 // Create TimeSliceThread for audio buffering
1039 juce::TimeSliceThread my_thread("Audio buffer thread");
1040
1041 // Start thread
1042 my_thread.startThread();
1043
1044 juce::AudioTransportSource transport1;
1045 transport1.setSource (my_source.get(),
1046 5000, // tells it to buffer this many samples ahead
1047 &my_thread,
1048 (double) sample_rate,
1049 audio->getNumChannels()); // sample rate of source
1050 transport1.setPosition (0);
1051 transport1.setGain(1.0);
1052
1053
1054 // Create MIXER
1055 juce::MixerAudioSource mixer;
1056 mixer.addInputSource(&transport1, false);
1057 audioSourcePlayer.setSource (&mixer);
1058
1059 // Start transports
1060 transport1.start();
1061
1062 while (transport1.isPlaying())
1063 {
1064 cout << "playing" << endl;
1065 std::this_thread::sleep_for(std::chrono::seconds(1));
1066 }
1067
1068 cout << "DONE!!!" << endl;
1069
1070 transport1.stop();
1071 transport1.setSource (0);
1072 audioSourcePlayer.setSource (0);
1073 my_thread.stopThread(500);
1074 deviceManager.removeAudioCallback (&audioSourcePlayer);
1075 deviceManager.closeAudioDevice();
1076 deviceManager.removeAllChangeListeners();
1077 deviceManager.dispatchPendingMessages();
1078
1079 cout << "End of Play()" << endl;
1080
1081
1082}
1083
1084// Clean up buffer after QImage is deleted
1085void Frame::cleanUpBuffer(void *info)
1086{
1087 if (info)
1088 {
1089 // Remove buffer since QImage tells us to
1090 unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1091 delete[] ptr_to_qbuffer;
1092 }
1093}
1094
1095// Add audio silence
1096void Frame::AddAudioSilence(int numSamples)
1097{
1098 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1099
1100 // Resize audio container
1101 audio->setSize(channels, numSamples, false, true, false);
1102 audio->clear();
1103 has_audio_data = true;
1104
1105 // Calculate max audio sample added
1106 if (numSamples > max_audio_sample)
1107 max_audio_sample = numSamples;
1108}
Header file for Frame class.
#define MAGICK_IMAGE_ALPHA(im, a)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
This class is used to resample audio data for many sequential frames.
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
This class represents a fraction.
Definition: Fraction.h:48
int num
Numerator for the fraction.
Definition: Fraction.h:50
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:94
int den
Denominator for the fraction.
Definition: Fraction.h:51
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:108
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:88
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1085
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:938
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:737
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:371
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:453
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:172
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:961
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:567
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:130
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:416
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:913
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:137
int GetWidth()
Get height of image.
Definition: Frame.cpp:548
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:554
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:849
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:97
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:325
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:986
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:924
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:255
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:1014
std::shared_ptr< juce::AudioSampleBuffer > audio
Definition: Frame.h:134
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:136
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:438
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:560
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1096
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:602
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:272
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:432
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:477
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:318
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:305
virtual ~Frame()
Destructor.
Definition: Frame.cpp:120
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:900
Frame()
Constructor - blank frame.
Definition: Frame.cpp:69
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:756
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:891
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:426
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:863
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:262
int GetHeight()
Get height of image.
Definition: Frame.cpp:542
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:499
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:506
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:135
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:952
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...