OpenShot Library | libopenshot  0.7.0
FFmpegReader.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <thread> // for std::this_thread::sleep_for
17 #include <chrono> // for std::chrono::milliseconds
18 #include <algorithm>
19 #include <cmath>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include "FFmpegUtilities.h"
24 #include "effects/CropHelpers.h"
25 
26 #include "FFmpegReader.h"
27 #include "Exceptions.h"
28 #include "MemoryTrim.h"
29 #include "Timeline.h"
30 #include "ZmqLogger.h"
31 
32 #define ENABLE_VAAPI 0
33 
34 #if USE_HW_ACCEL
35 #define MAX_SUPPORTED_WIDTH 1950
36 #define MAX_SUPPORTED_HEIGHT 1100
37 
38 #if ENABLE_VAAPI
39 #include "libavutil/hwcontext_vaapi.h"
40 
41 typedef struct VAAPIDecodeContext {
42  VAProfile va_profile;
43  VAEntrypoint va_entrypoint;
44  VAConfigID va_config;
45  VAContextID va_context;
46 
47 #if FF_API_STRUCT_VAAPI_CONTEXT
48  // FF_DISABLE_DEPRECATION_WARNINGS
49  int have_old_context;
50  struct vaapi_context *old_context;
51  AVBufferRef *device_ref;
52  // FF_ENABLE_DEPRECATION_WARNINGS
53 #endif
54 
55  AVHWDeviceContext *device;
56  AVVAAPIDeviceContext *hwctx;
57 
58  AVHWFramesContext *frames;
59  AVVAAPIFramesContext *hwfc;
60 
61  enum AVPixelFormat surface_format;
62  int surface_count;
63  } VAAPIDecodeContext;
64 #endif // ENABLE_VAAPI
65 #endif // USE_HW_ACCEL
66 
67 
68 using namespace openshot;
69 
70 int hw_de_on = 0;
71 #if USE_HW_ACCEL
72  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
73  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
74 #endif
75 
76 // Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts.
77 // swscale expects non-YUVJ formats plus explicit color-range metadata.
78 static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) {
79  switch (pix_fmt) {
80  case AV_PIX_FMT_YUVJ420P:
81  is_full_range = true;
82  return AV_PIX_FMT_YUV420P;
83  case AV_PIX_FMT_YUVJ422P:
84  is_full_range = true;
85  return AV_PIX_FMT_YUV422P;
86  case AV_PIX_FMT_YUVJ444P:
87  is_full_range = true;
88  return AV_PIX_FMT_YUV444P;
89  case AV_PIX_FMT_YUVJ440P:
90  is_full_range = true;
91  return AV_PIX_FMT_YUV440P;
92 #ifdef AV_PIX_FMT_YUVJ411P
93  case AV_PIX_FMT_YUVJ411P:
94  is_full_range = true;
95  return AV_PIX_FMT_YUV411P;
96 #endif
97  default:
98  return pix_fmt;
99  }
100 }
101 
102 FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
103  : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {}
104 
105 FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader)
106  : path(path), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
107  pStream(NULL), aStream(NULL), packet(NULL), pFrame(NULL), is_open(false), is_duration_known(false),
108  check_interlace(false), check_fps(false), duration_strategy(duration_strategy), previous_packet_location{-1, 0},
109  is_seeking(false), seeking_pts(0), seeking_frame(0), is_video_seek(true), seek_count(0),
110  seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0),
111  last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0),
112  hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0),
113  NO_PTS_OFFSET(-99999), enable_seek(true) {
114 
115  // Initialize FFMpeg, and register all formats and codecs
116  AV_REGISTER_ALL
117  AVCODEC_REGISTER_ALL
118 
119  // Init timestamp offsets
120  pts_offset_seconds = NO_PTS_OFFSET;
121  video_pts_seconds = NO_PTS_OFFSET;
122  audio_pts_seconds = NO_PTS_OFFSET;
123 
124  // Init cache
125  const int init_working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
126  const int init_final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
127  working_cache.SetMaxBytesFromInfo(init_working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
128  final_cache.SetMaxBytesFromInfo(init_final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
129 
130  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
131  if (inspect_reader) {
132  Open();
133  Close();
134  }
135 }
136 
137 FFmpegReader::~FFmpegReader() {
138  if (is_open)
139  // Auto close reader if not already done
140  Close();
141 }
142 
143 // This struct holds the associated video frame and starting sample # for an audio packet.
144 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
145  // Is frame even close to this one?
146  if (abs(location.frame - frame) >= 2)
147  // This is too far away to be considered
148  return false;
149 
150  // Note that samples_per_frame can vary slightly frame to frame when the
151  // audio sampling rate is not an integer multiple of the video fps.
152  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
153  if (abs(diff) <= amount)
154  // close
155  return true;
156 
157  // not close
158  return false;
159 }
160 
161 #if USE_HW_ACCEL
162 
163 // Get hardware pix format
164 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
165 {
166  const enum AVPixelFormat *p;
167 
168  // Prefer only the format matching the selected hardware decoder
169  int selected = openshot::Settings::Instance()->HARDWARE_DECODER;
170 
171  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
172  switch (*p) {
173 #if defined(__linux__)
174  // Linux pix formats
175  case AV_PIX_FMT_VAAPI:
176  if (selected == 1) {
177  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
178  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
179  return *p;
180  }
181  break;
182  case AV_PIX_FMT_VDPAU:
183  if (selected == 6) {
184  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
185  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
186  return *p;
187  }
188  break;
189 #endif
190 #if defined(_WIN32)
191  // Windows pix formats
192  case AV_PIX_FMT_DXVA2_VLD:
193  if (selected == 3) {
194  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
195  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
196  return *p;
197  }
198  break;
199  case AV_PIX_FMT_D3D11:
200  if (selected == 4) {
201  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
202  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
203  return *p;
204  }
205  break;
206 #endif
207 #if defined(__APPLE__)
208  // Apple pix formats
209  case AV_PIX_FMT_VIDEOTOOLBOX:
210  if (selected == 5) {
211  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
212  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
213  return *p;
214  }
215  break;
216 #endif
217  // Cross-platform pix formats
218  case AV_PIX_FMT_CUDA:
219  if (selected == 2) {
220  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
221  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
222  return *p;
223  }
224  break;
225  case AV_PIX_FMT_QSV:
226  if (selected == 7) {
227  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
228  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
229  return *p;
230  }
231  break;
232  default:
233  // This is only here to silence unused-enum warnings
234  break;
235  }
236  }
237  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
238  return AV_PIX_FMT_NONE;
239 }
240 
241 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
242 {
243  int ret;
244  switch (codecid) {
245  case AV_CODEC_ID_H264:
246  case AV_CODEC_ID_MPEG2VIDEO:
247  case AV_CODEC_ID_VC1:
248  case AV_CODEC_ID_WMV1:
249  case AV_CODEC_ID_WMV2:
250  case AV_CODEC_ID_WMV3:
251  ret = 1;
252  break;
253  default :
254  ret = 0;
255  break;
256  }
257  return ret;
258 }
259 #endif // USE_HW_ACCEL
260 
261 void FFmpegReader::Open() {
262  // Open reader if not already open
263  if (!is_open) {
264  // Prevent async calls to the following code
265  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
266 
267  // Initialize format context
268  pFormatCtx = NULL;
269  {
270  hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0);
271  hw_decode_failed = false;
272  hw_decode_error_count = 0;
273  hw_decode_succeeded = false;
274  ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
275  }
276 
277  // Open video file
278  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
279  throw InvalidFile("FFmpegReader could not open media file.", path);
280 
281  // Retrieve stream information
282  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
283  throw NoStreamsFound("No streams found in file.", path);
284 
285  videoStream = -1;
286  audioStream = -1;
287 
288  // Init end-of-file detection variables
289  packet_status.reset(true);
290 
291  // Loop through each stream, and identify the video and audio stream index
292  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
293  // Is this a video stream?
294  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
295  videoStream = i;
296  packet_status.video_eof = false;
297  packet_status.packets_eof = false;
298  packet_status.end_of_file = false;
299  }
300  // Is this an audio stream?
301  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
302  audioStream = i;
303  packet_status.audio_eof = false;
304  packet_status.packets_eof = false;
305  packet_status.end_of_file = false;
306  }
307  }
308  if (videoStream == -1 && audioStream == -1)
309  throw NoStreamsFound("No video or audio streams found in this file.", path);
310 
311  // Is there a video stream?
312  if (videoStream != -1) {
313  // Set the stream index
314  info.video_stream_index = videoStream;
315 
316  // Set the codec and codec context pointers
317  pStream = pFormatCtx->streams[videoStream];
318 
319  // Find the codec ID from stream
320  const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
321 
322  // Get codec and codec context from stream
323  const AVCodec *pCodec = avcodec_find_decoder(codecId);
324  AVDictionary *opts = NULL;
325  int retry_decode_open = 2;
326  // If hw accel is selected but hardware cannot handle repeat with software decoding
327  do {
328  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
329 #if USE_HW_ACCEL
330  if (hw_de_on && (retry_decode_open==2)) {
331  // Up to here no decision is made if hardware or software decode
332  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
333  }
334 #endif
335  retry_decode_open = 0;
336 
337  // Set number of threads equal to number of processors (not to exceed 16)
338  pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
339 
340  if (pCodec == NULL) {
341  throw InvalidCodec("A valid video codec could not be found for this file.", path);
342  }
343 
344  // Init options
345  av_dict_set(&opts, "strict", "experimental", 0);
346 #if USE_HW_ACCEL
347  if (hw_de_on && hw_de_supported) {
348  // Open Hardware Acceleration
349  int i_decoder_hw = 0;
350  char adapter[256];
351  char *adapter_ptr = NULL;
352  int adapter_num;
353  adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET;
354  ZmqLogger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num);
355 
356  // Set hardware pix format (callback)
357  pCodecCtx->get_format = get_hw_dec_format;
358 
359  if (adapter_num < 3 && adapter_num >=0) {
360 #if defined(__linux__)
361  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
362  adapter_ptr = adapter;
363  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
364  switch (i_decoder_hw) {
365  case 1:
366  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
367  break;
368  case 2:
369  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
370  break;
371  case 6:
372  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
373  break;
374  case 7:
375  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
376  break;
377  default:
378  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
379  break;
380  }
381 
382 #elif defined(_WIN32)
383  adapter_ptr = NULL;
384  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
385  switch (i_decoder_hw) {
386  case 2:
387  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
388  break;
389  case 3:
390  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
391  break;
392  case 4:
393  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
394  break;
395  case 7:
396  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
397  break;
398  default:
399  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
400  break;
401  }
402 #elif defined(__APPLE__)
403  adapter_ptr = NULL;
404  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
405  switch (i_decoder_hw) {
406  case 5:
407  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
408  break;
409  case 7:
410  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
411  break;
412  default:
413  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
414  break;
415  }
416 #endif
417 
418  } else {
419  adapter_ptr = NULL; // Just to be sure
420  }
421 
422  // Check if it is there and writable
423 #if defined(__linux__)
424  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
425 #elif defined(_WIN32)
426  if( adapter_ptr != NULL ) {
427 #elif defined(__APPLE__)
428  if( adapter_ptr != NULL ) {
429 #endif
430  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
431  }
432  else {
433  adapter_ptr = NULL; // use default
434  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
435  }
436 
437  hw_device_ctx = NULL;
438  // Here the first hardware initialisations are made
439  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
440  const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type);
441  std::string hw_msg = "HW decode active: ";
442  hw_msg += (hw_name ? hw_name : "unknown");
443  ZmqLogger::Instance()->Log(hw_msg);
444  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
445  throw InvalidCodec("Hardware device reference create failed.", path);
446  }
447 
448  /*
449  av_buffer_unref(&ist->hw_frames_ctx);
450  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
451  if (!ist->hw_frames_ctx) {
452  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
453  return AVERROR(ENOMEM);
454  }
455 
456  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
457 
458  frames_ctx->format = AV_PIX_FMT_CUDA;
459  frames_ctx->sw_format = avctx->sw_pix_fmt;
460  frames_ctx->width = avctx->width;
461  frames_ctx->height = avctx->height;
462 
463  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
464  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
465 
466 
467  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
468  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
469  if (ret < 0) {
470  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
471  return ret;
472  }
473  */
474  }
475  else {
476  ZmqLogger::Instance()->Log("HW decode active: no (falling back to software)");
477  throw InvalidCodec("Hardware device create failed.", path);
478  }
479  }
480 #endif // USE_HW_ACCEL
481 
482  // Disable per-frame threading for album arts
483  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
484  // but there's only one frame in this case.
485  if (HasAlbumArt())
486  {
487  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
488  }
489 
490  // Open video codec
491  int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
492  if (avcodec_return < 0) {
493  std::stringstream avcodec_error_msg;
494  avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
495  throw InvalidCodec(avcodec_error_msg.str(), path);
496  }
497 
498 #if USE_HW_ACCEL
499  if (hw_de_on && hw_de_supported) {
500  AVHWFramesConstraints *constraints = NULL;
501  void *hwconfig = NULL;
502  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
503 
504 // TODO: needs va_config!
505 #if ENABLE_VAAPI
506  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
507  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
508 #endif // ENABLE_VAAPI
509  if (constraints) {
510  if (pCodecCtx->coded_width < constraints->min_width ||
511  pCodecCtx->coded_height < constraints->min_height ||
512  pCodecCtx->coded_width > constraints->max_width ||
513  pCodecCtx->coded_height > constraints->max_height) {
514  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
515  hw_de_supported = 0;
516  retry_decode_open = 1;
517  AV_FREE_CONTEXT(pCodecCtx);
518  if (hw_device_ctx) {
519  av_buffer_unref(&hw_device_ctx);
520  hw_device_ctx = NULL;
521  }
522  }
523  else {
524  // All is just peachy
525  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
526  retry_decode_open = 0;
527  }
528  av_hwframe_constraints_free(&constraints);
529  if (hwconfig) {
530  av_freep(&hwconfig);
531  }
532  }
533  else {
534  int max_h, max_w;
535  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
536  max_h = openshot::Settings::Instance()->DE_LIMIT_HEIGHT_MAX;
537  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
538  max_w = openshot::Settings::Instance()->DE_LIMIT_WIDTH_MAX;
539  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
540  //cerr << "Constraints could not be found using default limit\n";
541  if (pCodecCtx->coded_width < 0 ||
542  pCodecCtx->coded_height < 0 ||
543  pCodecCtx->coded_width > max_w ||
544  pCodecCtx->coded_height > max_h ) {
545  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
546  hw_de_supported = 0;
547  retry_decode_open = 1;
548  AV_FREE_CONTEXT(pCodecCtx);
549  if (hw_device_ctx) {
550  av_buffer_unref(&hw_device_ctx);
551  hw_device_ctx = NULL;
552  }
553  }
554  else {
555  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
556  retry_decode_open = 0;
557  }
558  }
559  } // if hw_de_on && hw_de_supported
560  else {
561  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
562  }
563 #else
564  retry_decode_open = 0;
565 #endif // USE_HW_ACCEL
566  } while (retry_decode_open); // retry_decode_open
567  // Free options
568  av_dict_free(&opts);
569 
570  // Update the File Info struct with video details (if a video stream is found)
571  UpdateVideoInfo();
572  }
573 
574  // Is there an audio stream?
575  if (audioStream != -1) {
576  // Set the stream index
577  info.audio_stream_index = audioStream;
578 
579  // Get a pointer to the codec context for the audio stream
580  aStream = pFormatCtx->streams[audioStream];
581 
582  // Find the codec ID from stream
583  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
584 
585  // Get codec and codec context from stream
586  const AVCodec *aCodec = avcodec_find_decoder(codecId);
587  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
588 
589  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
590  aCodecCtx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
591 
592  bool audio_opened = false;
593  if (aCodec != NULL) {
594  // Init options
595  AVDictionary *opts = NULL;
596  av_dict_set(&opts, "strict", "experimental", 0);
597 
598  // Open audio codec
599  audio_opened = (avcodec_open2(aCodecCtx, aCodec, &opts) >= 0);
600 
601  // Free options
602  av_dict_free(&opts);
603  }
604 
605  if (audio_opened) {
606  // Update the File Info struct with audio details (if an audio stream is found)
607  UpdateAudioInfo();
608 
609  // Disable malformed audio stream metadata (prevents divide-by-zero / invalid resampling math)
610  const bool invalid_audio_info =
611  (info.channels <= 0) ||
612  (info.sample_rate <= 0) ||
613  (info.audio_timebase.num <= 0) ||
614  (info.audio_timebase.den <= 0) ||
615  (aCodecCtx->sample_fmt == AV_SAMPLE_FMT_NONE);
616  if (invalid_audio_info) {
617  ZmqLogger::Instance()->AppendDebugMethod(
618  "FFmpegReader::Open (Disable invalid audio stream)",
619  "channels", info.channels,
620  "sample_rate", info.sample_rate,
621  "audio_timebase.num", info.audio_timebase.num,
622  "audio_timebase.den", info.audio_timebase.den,
623  "sample_fmt", static_cast<int>(aCodecCtx ? aCodecCtx->sample_fmt : AV_SAMPLE_FMT_NONE));
624  info.has_audio = false;
625  info.audio_stream_index = -1;
626  audioStream = -1;
627  packet_status.audio_eof = true;
628  if (aCodecCtx) {
629  if (avcodec_is_open(aCodecCtx)) {
630  avcodec_flush_buffers(aCodecCtx);
631  }
632  AV_FREE_CONTEXT(aCodecCtx);
633  aCodecCtx = nullptr;
634  }
635  aStream = nullptr;
636  }
637  } else {
638  // Keep decoding video, but disable bad/unsupported audio stream.
639  ZmqLogger::Instance()->AppendDebugMethod(
640  "FFmpegReader::Open (Audio codec unavailable; disabling audio)",
641  "audioStream", audioStream);
642  info.has_audio = false;
643  info.audio_stream_index = -1;
644  audioStream = -1;
645  packet_status.audio_eof = true;
646  if (aCodecCtx) {
647  AV_FREE_CONTEXT(aCodecCtx);
648  aCodecCtx = nullptr;
649  }
650  aStream = nullptr;
651  }
652  }
653 
654  // Guard invalid frame-rate / timebase values from malformed streams.
655  if (info.fps.num <= 0 || info.fps.den <= 0) {
656  ZmqLogger::Instance()->AppendDebugMethod(
657  "FFmpegReader::Open (Invalid FPS detected; applying fallback)",
658  "fps.num", info.fps.num,
659  "fps.den", info.fps.den);
660  info.fps.num = 30;
661  info.fps.den = 1;
662  }
663  if (info.video_timebase.num <= 0 || info.video_timebase.den <= 0) {
664  ZmqLogger::Instance()->AppendDebugMethod(
665  "FFmpegReader::Open (Invalid video_timebase detected; applying fallback)",
666  "video_timebase.num", info.video_timebase.num,
667  "video_timebase.den", info.video_timebase.den);
668  info.video_timebase = info.fps.Reciprocal();
669  }
670 
671  // Add format metadata (if any)
672  AVDictionaryEntry *tag = NULL;
673  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
674  QString str_key = tag->key;
675  QString str_value = tag->value;
676  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
677  }
678 
679  // Process video stream side data (rotation, spherical metadata, etc)
680  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
681  AVStream* st = pFormatCtx->streams[i];
682  if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
683  size_t side_data_size = 0;
684  const uint8_t *displaymatrix = ffmpeg_stream_get_side_data(
685  st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size);
686  if (displaymatrix &&
687  side_data_size >= 9 * sizeof(int32_t) &&
688  !info.metadata.count("rotate")) {
689  double rotation = -av_display_rotation_get(
690  reinterpret_cast<const int32_t *>(displaymatrix));
691  if (std::isnan(rotation))
692  rotation = 0;
693  info.metadata["rotate"] = std::to_string(rotation);
694  }
695 
696  const uint8_t *spherical = ffmpeg_stream_get_side_data(
697  st, AV_PKT_DATA_SPHERICAL, &side_data_size);
698  if (spherical && side_data_size >= sizeof(AVSphericalMapping)) {
699  info.metadata["spherical"] = "1";
700 
701  const AVSphericalMapping *map =
702  reinterpret_cast<const AVSphericalMapping *>(spherical);
703  const char *proj_name = av_spherical_projection_name(map->projection);
704  info.metadata["spherical_projection"] = proj_name ? proj_name : "unknown";
705 
706  auto to_deg = [](int32_t v) {
707  return static_cast<double>(v) / 65536.0;
708  };
709  info.metadata["spherical_yaw"] = std::to_string(to_deg(map->yaw));
710  info.metadata["spherical_pitch"] = std::to_string(to_deg(map->pitch));
711  info.metadata["spherical_roll"] = std::to_string(to_deg(map->roll));
712  }
713  break;
714  }
715  }
716 
717  // Init previous audio location to zero
718  previous_packet_location.frame = -1;
719  previous_packet_location.sample_start = 0;
720 
721  // Adjust cache size based on size of frame and audio
722  const int working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, int(OPEN_MP_NUM_PROCESSORS * info.fps.ToDouble() * 2));
723  const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2);
724  working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
725  final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
726 
727  // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
728  // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
729  UpdatePTSOffset();
730 
731  // Override an invalid framerate
732  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
733  // Calculate FPS, duration, video bit rate, and video length manually
734  // by scanning through all the video stream packets
735  CheckFPS();
736  }
737 
738  // Mark as "open"
739  is_open = true;
740 
741  // Seek back to beginning of file (if not already seeking)
742  if (!is_seeking) {
743  Seek(1);
744  }
745  }
746 }
747 
748 void FFmpegReader::Close() {
749  // Close all objects, if reader is 'open'
750  if (is_open) {
751  // Prevent async calls to the following code
752  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
753 
754  // Mark as "closed"
755  is_open = false;
756 
757  // Keep track of most recent packet
758  AVPacket *recent_packet = packet;
759 
760  // Drain any packets from the decoder
761  packet = NULL;
762  int attempts = 0;
763  int max_attempts = 128;
764  while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
765  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
766  "packets_read", packet_status.packets_read(),
767  "packets_decoded", packet_status.packets_decoded(),
768  "attempts", attempts);
769  if (packet_status.video_decoded < packet_status.video_read) {
770  ProcessVideoPacket(info.video_length);
771  }
772  if (packet_status.audio_decoded < packet_status.audio_read) {
773  ProcessAudioPacket(info.video_length);
774  }
775  attempts++;
776  }
777 
778  // Remove packet
779  if (recent_packet) {
780  RemoveAVPacket(recent_packet);
781  }
782 
783  // Close the video codec
784  if (info.has_video) {
785  if(avcodec_is_open(pCodecCtx)) {
786  avcodec_flush_buffers(pCodecCtx);
787  }
788  AV_FREE_CONTEXT(pCodecCtx);
789 #if USE_HW_ACCEL
790  if (hw_de_on) {
791  if (hw_device_ctx) {
792  av_buffer_unref(&hw_device_ctx);
793  hw_device_ctx = NULL;
794  }
795  }
796 #endif // USE_HW_ACCEL
797  if (img_convert_ctx) {
798  sws_freeContext(img_convert_ctx);
799  img_convert_ctx = nullptr;
800  }
801  if (pFrameRGB_cached) {
802  AV_FREE_FRAME(&pFrameRGB_cached);
803  }
804  }
805 
806  // Close the audio codec
807  if (info.has_audio) {
808  if(avcodec_is_open(aCodecCtx)) {
809  avcodec_flush_buffers(aCodecCtx);
810  }
811  AV_FREE_CONTEXT(aCodecCtx);
812  if (avr_ctx) {
813  SWR_CLOSE(avr_ctx);
814  SWR_FREE(&avr_ctx);
815  avr_ctx = nullptr;
816  }
817  }
818 
819  // Clear final cache
820  final_cache.Clear();
821  working_cache.Clear();
822 
823  // Close the video file
824  avformat_close_input(&pFormatCtx);
825  av_freep(&pFormatCtx);
826 
827  // Do not trim here; trimming is handled on explicit cache clears
828 
829  // Reset some variables
830  last_frame = 0;
831  hold_packet = false;
832  largest_frame_processed = 0;
833  seek_audio_frame_found = 0;
834  seek_video_frame_found = 0;
835  current_video_frame = 0;
836  last_video_frame.reset();
837  last_final_video_frame.reset();
838  }
839 }
840 
841 bool FFmpegReader::HasAlbumArt() {
842  // Check if the video stream we use is an attached picture
843  // This won't return true if the file has a cover image as a secondary stream
844  // like an MKV file with an attached image file
845  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
846  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
847 }
848 
849 double FFmpegReader::PickDurationSeconds() const {
850  auto has_value = [](double value) { return value > 0.0; };
851 
852  switch (duration_strategy) {
853  case DurationStrategy::VideoPreferred:
854  if (has_value(video_stream_duration_seconds))
855  return video_stream_duration_seconds;
856  if (has_value(audio_stream_duration_seconds))
857  return audio_stream_duration_seconds;
858  if (has_value(format_duration_seconds))
859  return format_duration_seconds;
860  break;
861  case DurationStrategy::AudioPreferred:
862  if (has_value(audio_stream_duration_seconds))
863  return audio_stream_duration_seconds;
864  if (has_value(video_stream_duration_seconds))
865  return video_stream_duration_seconds;
866  if (has_value(format_duration_seconds))
867  return format_duration_seconds;
868  break;
869  case DurationStrategy::LongestStream:
870  default:
871  {
872  double longest = 0.0;
873  if (has_value(video_stream_duration_seconds))
874  longest = std::max(longest, video_stream_duration_seconds);
875  if (has_value(audio_stream_duration_seconds))
876  longest = std::max(longest, audio_stream_duration_seconds);
877  if (has_value(format_duration_seconds))
878  longest = std::max(longest, format_duration_seconds);
879  if (has_value(longest))
880  return longest;
881  }
882  break;
883  }
884 
885  if (has_value(format_duration_seconds))
886  return format_duration_seconds;
887  if (has_value(inferred_duration_seconds))
888  return inferred_duration_seconds;
889 
890  return 0.0;
891 }
892 
893 void FFmpegReader::ApplyDurationStrategy() {
894  const double fps_value = info.fps.ToDouble();
895  const double chosen_seconds = PickDurationSeconds();
896 
897  if (chosen_seconds <= 0.0 || fps_value <= 0.0) {
898  info.duration = 0.0f;
899  info.video_length = 0;
900  is_duration_known = false;
901  return;
902  }
903 
904  const int64_t frames = static_cast<int64_t>(std::llround(chosen_seconds * fps_value));
905  if (frames <= 0) {
906  info.duration = 0.0f;
907  info.video_length = 0;
908  is_duration_known = false;
909  return;
910  }
911 
912  info.video_length = frames;
913  info.duration = static_cast<float>(static_cast<double>(frames) / fps_value);
914  is_duration_known = true;
915 }
916 
917 void FFmpegReader::UpdateAudioInfo() {
918  int codec_channels =
919 #if HAVE_CH_LAYOUT
920  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
921 #else
922  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
923 #endif
924 
925  // Set default audio channel layout (if needed)
926 #if HAVE_CH_LAYOUT
927  AVChannelLayout audio_ch_layout = ffmpeg_get_valid_channel_layout(
928  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, codec_channels);
929  if (audio_ch_layout.nb_channels > 0) {
930  av_channel_layout_uninit(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout));
931  av_channel_layout_copy(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout), &audio_ch_layout);
932  codec_channels = audio_ch_layout.nb_channels;
933  }
934 #else
935  if (codec_channels > 0 && AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
936  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
937 #endif
938 
939  if (info.sample_rate > 0) {
940  // Skip init - if info struct already populated
941  return;
942  }
943 
944  auto record_duration = [](double &target, double seconds) {
945  if (seconds > 0.0)
946  target = std::max(target, seconds);
947  };
948 
949  // Set values of FileInfo struct
950  info.has_audio = true;
951  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
952  info.acodec = aCodecCtx->codec->name;
953 #if HAVE_CH_LAYOUT
954  info.channels = audio_ch_layout.nb_channels;
955  info.channel_layout = static_cast<ChannelLayout>(ffmpeg_channel_layout_mask(audio_ch_layout));
956 #else
957  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
958  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
959 #endif
960 
961  // If channel layout is not set, guess based on the number of channels
962  if (info.channel_layout == 0) {
963  if (info.channels == 1) {
964  info.channel_layout = openshot::LAYOUT_MONO;
965  } else if (info.channels == 2) {
966  info.channel_layout = openshot::LAYOUT_STEREO;
967  }
968  }
969 
970  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
971  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
972  if (info.audio_bit_rate <= 0) {
973  // Get bitrate from format
974  info.audio_bit_rate = pFormatCtx->bit_rate;
975  }
976 
977  // Set audio timebase
978  info.audio_timebase.num = aStream->time_base.num;
979  info.audio_timebase.den = aStream->time_base.den;
980 
981  // Get timebase of audio stream (if valid) and greater than the current duration
982  if (aStream->duration > 0) {
983  record_duration(audio_stream_duration_seconds, aStream->duration * info.audio_timebase.ToDouble());
984  }
985  if (pFormatCtx->duration > 0) {
986  // Use the format's duration when stream duration is missing or shorter
987  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
988  }
989 
990  // Calculate duration from filesize and bitrate (if any)
991  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
992  // Estimate from bitrate, total bytes, and framerate
993  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
994  }
995 
996  // Set video timebase (if no video stream was found)
997  if (!info.has_video) {
998  // Set a few important default video settings (so audio can be divided into frames)
999  info.fps.num = 30;
1000  info.fps.den = 1;
1001  info.video_timebase.num = 1;
1002  info.video_timebase.den = 30;
1003  if (info.width <= 0 || info.height <= 0) {
1004  info.width = 720;
1005  info.height = 480;
1006  }
1007 
1008  // Use timeline to set correct width & height (if any)
1009  Clip *parent = static_cast<Clip *>(ParentClip());
1010  if (parent) {
1011  if (parent->ParentTimeline()) {
1012  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1013  info.width = parent->ParentTimeline()->preview_width;
1014  info.height = parent->ParentTimeline()->preview_height;
1015  }
1016  }
1017  }
1018 
1019  ApplyDurationStrategy();
1020 
1021  // Add audio metadata (if any found)
1022  AVDictionaryEntry *tag = NULL;
1023  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1024  QString str_key = tag->key;
1025  QString str_value = tag->value;
1026  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1027  }
1028 #if HAVE_CH_LAYOUT
1029  av_channel_layout_uninit(&audio_ch_layout);
1030 #endif
1031 }
1032 
1033 void FFmpegReader::UpdateVideoInfo() {
1034  if (info.vcodec.length() > 0) {
1035  // Skip init - if info struct already populated
1036  return;
1037  }
1038 
1039  auto record_duration = [](double &target, double seconds) {
1040  if (seconds > 0.0)
1041  target = std::max(target, seconds);
1042  };
1043 
1044  // Set values of FileInfo struct
1045  info.has_video = true;
1046  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
1047  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
1048  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
1049  info.vcodec = pCodecCtx->codec->name;
1050  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
1051 
1052  // Frame rate from the container and codec
1053  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
1054  if (!check_fps) {
1055  info.fps.num = framerate.num;
1056  info.fps.den = framerate.den;
1057  }
1058 
1059  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
1060 
1061  // TODO: remove excessive debug info in the next releases
1062  // The debug info below is just for comparison and troubleshooting on users side during the transition period
1063  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
1064 
1065  if (pStream->sample_aspect_ratio.num != 0) {
1066  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
1067  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
1068  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
1069  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
1070  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
1071  } else {
1072  info.pixel_ratio.num = 1;
1073  info.pixel_ratio.den = 1;
1074  }
1075  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1076 
1077  // Calculate the DAR (display aspect ratio)
1078  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
1079 
1080  // Reduce size fraction
1081  size.Reduce();
1082 
1083  // Set the ratio based on the reduced fraction
1084  info.display_ratio.num = size.num;
1085  info.display_ratio.den = size.den;
1086 
1087  // Get scan type and order from codec context/params
1088  if (!check_interlace) {
1089  check_interlace = true;
1090  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
1091  switch(field_order) {
1092  case AV_FIELD_PROGRESSIVE:
1093  info.interlaced_frame = false;
1094  break;
1095  case AV_FIELD_TT:
1096  case AV_FIELD_TB:
1097  info.interlaced_frame = true;
1098  info.top_field_first = true;
1099  break;
1100  case AV_FIELD_BT:
1101  case AV_FIELD_BB:
1102  info.interlaced_frame = true;
1103  info.top_field_first = false;
1104  break;
1105  case AV_FIELD_UNKNOWN:
1106  // Check again later?
1107  check_interlace = false;
1108  break;
1109  }
1110  // check_interlace will prevent these checks being repeated,
1111  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
1112  }
1113 
1114  // Set the video timebase
1115  info.video_timebase.num = pStream->time_base.num;
1116  info.video_timebase.den = pStream->time_base.den;
1117 
1118  // Set the duration in seconds, and video length (# of frames)
1119  record_duration(video_stream_duration_seconds, pStream->duration * info.video_timebase.ToDouble());
1120 
1121  // Check for valid duration (if found)
1122  if (pFormatCtx->duration >= 0) {
1123  // Use the format's duration as another candidate
1124  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
1125  }
1126 
1127  // Calculate duration from filesize and bitrate (if any)
1128  if (info.video_bit_rate > 0 && info.file_size > 0) {
1129  // Estimate from bitrate, total bytes, and framerate
1130  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
1131  }
1132 
1133  // Certain "image" formats do not have a valid duration
1134  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1135  pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
1136  // Force an "image" duration
1137  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1138  info.has_single_image = true;
1139  }
1140  // Static GIFs can have no usable duration; fall back to a small default
1141  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1142  pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) {
1143  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1144  info.has_single_image = true;
1145  }
1146 
1147  ApplyDurationStrategy();
1148 
1149  // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior.
1150  // This keeps timing/flags consistent regardless of which reader path was used.
1151  if (!info.has_single_image) {
1152  const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream);
1153  const bool likely_still_codec =
1154  codec_id == AV_CODEC_ID_MJPEG ||
1155  codec_id == AV_CODEC_ID_PNG ||
1156  codec_id == AV_CODEC_ID_BMP ||
1157  codec_id == AV_CODEC_ID_TIFF ||
1158  codec_id == AV_CODEC_ID_WEBP ||
1159  codec_id == AV_CODEC_ID_JPEG2000;
1160  const bool likely_image_demuxer =
1161  pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name &&
1162  strstr(pFormatCtx->iformat->name, "image2");
1163  const bool has_attached_pic = HasAlbumArt();
1164  const bool single_frame_stream =
1165  (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1);
1166  const bool single_frame_clip = info.video_length <= 1;
1167 
1168  const bool is_still_image_video =
1169  has_attached_pic ||
1170  ((single_frame_stream || single_frame_clip) &&
1171  (likely_still_codec || likely_image_demuxer));
1172 
1173  if (is_still_image_video) {
1174  info.has_single_image = true;
1175 
1176  // Only force long duration for standalone images. For audio + attached-art
1177  // files, keep stream-derived duration so the cover image spans the audio.
1178  if (audioStream < 0) {
1179  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1180  }
1181 
1182  ApplyDurationStrategy();
1183  }
1184  }
1185 
1186  // Add video metadata (if any)
1187  AVDictionaryEntry *tag = NULL;
1188  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1189  QString str_key = tag->key;
1190  QString str_value = tag->value;
1191  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1192  }
1193 }
1194 
1195 bool FFmpegReader::GetIsDurationKnown() {
1196  return this->is_duration_known;
1197 }
1198 
1199 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
1200  last_seek_max_frame = -1;
1201  seek_stagnant_count = 0;
1202  // Check for open reader (or throw exception)
1203  if (!is_open)
1204  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
1205 
1206  // Adjust for a requested frame that is too small or too large
1207  if (requested_frame < 1)
1208  requested_frame = 1;
1209  if (requested_frame > info.video_length && is_duration_known)
1210  requested_frame = info.video_length;
1211  if (info.has_video && info.video_length == 0)
1212  // Invalid duration of video file
1213  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
1214 
1215  // Debug output
1216  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
1217 
1218  // Check the cache for this frame
1219  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1220  if (frame) {
1221  // Debug output
1222  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
1223  // Return the cached frame
1224  return frame;
1225  } else {
1226 
1227  // Prevent async calls to the remainder of this code
1228  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
1229 
1230  // Check the cache a 2nd time (due to the potential previous lock)
1231  frame = final_cache.GetFrame(requested_frame);
1232  if (frame) {
1233  // Debug output
1234  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
1235  } else {
1236  // Frame is not in cache
1237  // Reset seek count
1238  seek_count = 0;
1239 
1240  // Are we within X frames of the requested frame?
1241  int64_t diff = requested_frame - last_frame;
1242  if (diff >= 1 && diff <= 20) {
1243  // Continue walking the stream
1244  frame = ReadStream(requested_frame);
1245  } else {
1246  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
1247  if (enable_seek) {
1248  // Only seek if enabled
1249  Seek(requested_frame);
1250 
1251  } else if (!enable_seek && diff < 0) {
1252  // Start over, since we can't seek, and the requested frame is smaller than our position
1253  // Since we are seeking to frame 1, this actually just closes/re-opens the reader
1254  Seek(1);
1255  }
1256 
1257  // Then continue walking the stream
1258  frame = ReadStream(requested_frame);
1259  }
1260  }
1261  return frame;
1262  }
1263 }
1264 
1265 // Read the stream until we find the requested Frame
1266 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
1267  // Allocate video frame
1268  bool check_seek = false;
1269  int packet_error = -1;
1270  int64_t no_progress_count = 0;
1271  int64_t prev_packets_read = packet_status.packets_read();
1272  int64_t prev_packets_decoded = packet_status.packets_decoded();
1273  int64_t prev_video_decoded = packet_status.video_decoded;
1274  double prev_video_pts_seconds = video_pts_seconds;
1275 
1276  // Debug output
1277  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame);
1278 
1279  // Loop through the stream until the correct frame is found
1280  while (true) {
1281  // Check if working frames are 'finished'
1282  if (!is_seeking) {
1283  // Check for final frames
1284  CheckWorkingFrames(requested_frame);
1285  }
1286 
1287  // Check if requested 'final' frame is available (and break out of loop if found)
1288  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1289  if (is_cache_found) {
1290  break;
1291  }
1292 
1293  if (!hold_packet || !packet) {
1294  // Get the next packet
1295  packet_error = GetNextPacket();
1296  if (packet_error < 0 && !packet) {
1297  // No more packets to be found
1298  packet_status.packets_eof = true;
1299  }
1300  }
1301 
1302  // Debug output
1303  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
1304 
1305  // Check the status of a seek (if any)
1306  if (is_seeking) {
1307  check_seek = CheckSeek();
1308  } else {
1309  check_seek = false;
1310  }
1311 
1312  if (check_seek) {
1313  // Packet may become NULL on Close inside Seek if CheckSeek returns false
1314  // Jump to the next iteration of this loop
1315  continue;
1316  }
1317 
1318  // Video packet
1319  if ((info.has_video && packet && packet->stream_index == videoStream) ||
1320  (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1321  (info.has_video && !packet && !packet_status.video_eof)) {
1322  // Process Video Packet
1323  ProcessVideoPacket(requested_frame);
1324  if (ReopenWithoutHardwareDecode(requested_frame)) {
1325  continue;
1326  }
1327  }
1328  // Audio packet
1329  if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1330  (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1331  (info.has_audio && !packet && !packet_status.audio_eof)) {
1332  // Process Audio Packet
1333  ProcessAudioPacket(requested_frame);
1334  }
1335 
1336  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1337  // if the has_video or has_audio properties are manually overridden)
1338  if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1339  (!info.has_audio && packet && packet->stream_index == audioStream)) {
1340  // Keep track of deleted packet counts
1341  if (packet->stream_index == videoStream) {
1342  packet_status.video_decoded++;
1343  } else if (packet->stream_index == audioStream) {
1344  packet_status.audio_decoded++;
1345  }
1346 
1347  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1348  // if the has_video or has_audio properties are manually overridden)
1349  RemoveAVPacket(packet);
1350  packet = NULL;
1351  }
1352 
1353  // Determine end-of-stream (waiting until final decoder threads finish)
1354  // Force end-of-stream in some situations
1355  packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1356  if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1357  // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1358  // If we have no more packets, and all known packets have been decoded
1359  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1360  if (!packet_status.video_eof) {
1361  packet_status.video_eof = true;
1362  }
1363  if (!packet_status.audio_eof) {
1364  packet_status.audio_eof = true;
1365  }
1366  packet_status.end_of_file = true;
1367  break;
1368  }
1369 
1370  // Detect decoder stalls with no progress at EOF and force completion so
1371  // missing frames can be finalized from prior image data.
1372  const bool has_progress =
1373  (packet_status.packets_read() != prev_packets_read) ||
1374  (packet_status.packets_decoded() != prev_packets_decoded) ||
1375  (packet_status.video_decoded != prev_video_decoded) ||
1376  (video_pts_seconds != prev_video_pts_seconds);
1377 
1378  if (has_progress) {
1379  no_progress_count = 0;
1380  } else {
1381  no_progress_count++;
1382  if (no_progress_count >= 2000
1383  && packet_status.packets_eof
1384  && !packet
1385  && !hold_packet) {
1386  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF after stall)",
1387  "requested_frame", requested_frame,
1388  "no_progress_count", no_progress_count,
1389  "packets_read", packet_status.packets_read(),
1390  "packets_decoded", packet_status.packets_decoded(),
1391  "video_decoded", packet_status.video_decoded,
1392  "audio_decoded", packet_status.audio_decoded);
1393  packet_status.video_eof = true;
1394  packet_status.audio_eof = true;
1395  packet_status.end_of_file = true;
1396  break;
1397  }
1398  }
1399  prev_packets_read = packet_status.packets_read();
1400  prev_packets_decoded = packet_status.packets_decoded();
1401  prev_video_decoded = packet_status.video_decoded;
1402  prev_video_pts_seconds = video_pts_seconds;
1403  } // end while
1404 
1405  // Debug output
1406  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1407  "packets_read", packet_status.packets_read(),
1408  "packets_decoded", packet_status.packets_decoded(),
1409  "end_of_file", packet_status.end_of_file,
1410  "largest_frame_processed", largest_frame_processed,
1411  "Working Cache Count", working_cache.Count());
1412 
1413  // Have we reached end-of-stream (or the final frame)?
1414  if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1415  // Force end-of-stream
1416  packet_status.end_of_file = true;
1417  }
1418  if (packet_status.end_of_file) {
1419  // Mark any other working frames as 'finished'
1420  CheckWorkingFrames(requested_frame);
1421  }
1422 
1423  // Return requested frame (if found)
1424  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1425  if (frame)
1426  // Return prepared frame
1427  return frame;
1428  else {
1429 
1430  // Check if largest frame is still cached
1431  frame = final_cache.GetFrame(largest_frame_processed);
1432  int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1433  info.sample_rate, info.channels);
1434  if (frame) {
1435  // Copy and return the largest processed frame (assuming it was the last in the video file)
1436  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1437  if (frame->has_image_data) {
1438  f->AddImage(std::make_shared<QImage>(frame->GetImage()->copy()));
1439  }
1440 
1441  // Use solid color (if no image data found)
1442  if (!frame->has_image_data) {
1443  // Use solid black frame if no image data available
1444  f->AddColor(info.width, info.height, "#000");
1445  }
1446  // Silence audio data (if any), since we are repeating the last frame
1447  f->AddAudioSilence(samples_in_frame);
1448 
1449  return f;
1450  } else {
1451  // The largest processed frame is no longer in cache. Prefer the most recent
1452  // finalized image first, then decoded image, to avoid black flashes.
1453  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1454  if (last_final_video_frame && last_final_video_frame->has_image_data
1455  && last_final_video_frame->number <= requested_frame) {
1456  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
1457  } else if (last_video_frame && last_video_frame->has_image_data
1458  && last_video_frame->number <= requested_frame) {
1459  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
1460  } else {
1461  f->AddColor(info.width, info.height, "#000");
1462  }
1463  f->AddAudioSilence(samples_in_frame);
1464  return f;
1465  }
1466  }
1467 
1468 }
1469 
1470 // Get the next packet (if any)
1471 int FFmpegReader::GetNextPacket() {
1472  int found_packet = 0;
1473  AVPacket *next_packet;
1474  next_packet = new AVPacket();
1475  found_packet = av_read_frame(pFormatCtx, next_packet);
1476 
1477  if (packet) {
1478  // Remove previous packet before getting next one
1479  RemoveAVPacket(packet);
1480  packet = NULL;
1481  }
1482  if (found_packet >= 0) {
1483  // Update current packet pointer
1484  packet = next_packet;
1485 
1486  // Keep track of packet stats
1487  if (packet->stream_index == videoStream) {
1488  packet_status.video_read++;
1489  } else if (packet->stream_index == audioStream) {
1490  packet_status.audio_read++;
1491  }
1492  } else {
1493  // No more packets found
1494  delete next_packet;
1495  packet = NULL;
1496  }
1497  // Return if packet was found (or error number)
1498  return found_packet;
1499 }
1500 
1501 // Get an AVFrame (if any)
1502 bool FFmpegReader::GetAVFrame() {
1503  int frameFinished = 0;
1504  auto note_hw_decode_failure = [&](int err, const char* stage) {
1505 #if USE_HW_ACCEL
1506  if (!hw_de_on || !hw_de_supported || force_sw_decode) {
1507  return;
1508  }
1509  if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) {
1510  hw_decode_error_count++;
1511  ZmqLogger::Instance()->AppendDebugMethod(
1512  std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")",
1513  "error_count", hw_decode_error_count,
1514  "error", err);
1515  if (hw_decode_error_count >= 3) {
1516  hw_decode_failed = true;
1517  }
1518  }
1519 #else
1520  (void) err;
1521  (void) stage;
1522 #endif
1523  };
1524 
1525  // Decode video frame
1526  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1527 
1528 #if IS_FFMPEG_3_2
1529  int send_packet_err = 0;
1530  int64_t send_packet_pts = 0;
1531  if ((packet && packet->stream_index == videoStream) || !packet) {
1532  send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1533 
1534  if (packet && send_packet_err >= 0) {
1535  send_packet_pts = GetPacketPTS();
1536  hold_packet = false;
1537  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1538  }
1539  }
1540 
1541  #if USE_HW_ACCEL
1542  // Get the format from the variables set in get_hw_dec_format
1543  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1544  hw_de_av_device_type = hw_de_av_device_type_global;
1545  #endif // USE_HW_ACCEL
1546  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1547  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1548  note_hw_decode_failure(send_packet_err, "send_packet");
1549  if (send_packet_err == AVERROR(EAGAIN)) {
1550  hold_packet = true;
1551  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1552  }
1553  if (send_packet_err == AVERROR(EINVAL)) {
1554  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1555  }
1556  if (send_packet_err == AVERROR(ENOMEM)) {
1557  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1558  }
1559  }
1560 
1561  // Always try and receive a packet, if not EOF.
1562  // Even if the above avcodec_send_packet failed to send,
1563  // we might still need to receive a packet.
1564  int receive_frame_err = 0;
1565  AVFrame *decoded_frame = next_frame;
1566  AVFrame *next_frame2;
1567 #if USE_HW_ACCEL
1568  if (hw_de_on && hw_de_supported) {
1569  next_frame2 = AV_ALLOCATE_FRAME();
1570  }
1571  else
1572 #endif // USE_HW_ACCEL
1573  {
1574  next_frame2 = next_frame;
1575  }
1576  pFrame = AV_ALLOCATE_FRAME();
1577  while (receive_frame_err >= 0) {
1578  receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1579 
1580  if (receive_frame_err != 0) {
1581  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1582  note_hw_decode_failure(receive_frame_err, "receive_frame");
1583 
1584  if (receive_frame_err == AVERROR_EOF) {
1585  ZmqLogger::Instance()->AppendDebugMethod(
1586  "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1587  avcodec_flush_buffers(pCodecCtx);
1588  packet_status.video_eof = true;
1589  }
1590  if (receive_frame_err == AVERROR(EINVAL)) {
1591  ZmqLogger::Instance()->AppendDebugMethod(
1592  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1593  avcodec_flush_buffers(pCodecCtx);
1594  }
1595  if (receive_frame_err == AVERROR(EAGAIN)) {
1596  ZmqLogger::Instance()->AppendDebugMethod(
1597  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1598  }
1599  if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1600  ZmqLogger::Instance()->AppendDebugMethod(
1601  "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1602  }
1603 
1604  // Break out of decoding loop
1605  // Nothing ready for decoding yet
1606  break;
1607  }
1608 
1609 #if USE_HW_ACCEL
1610  if (hw_de_on && hw_de_supported) {
1611  int err;
1612  if (next_frame2->format == hw_de_av_pix_fmt) {
1613  if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) {
1614  ZmqLogger::Instance()->AppendDebugMethod(
1615  "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)",
1616  "hw_de_on", hw_de_on,
1617  "error", err);
1618  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer");
1619  break;
1620  }
1621  if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) {
1622  ZmqLogger::Instance()->AppendDebugMethod(
1623  "FFmpegReader::GetAVFrame (Failed to copy props to output frame)",
1624  "hw_de_on", hw_de_on,
1625  "error", err);
1626  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props");
1627  break;
1628  }
1629  if (next_frame->format == AV_PIX_FMT_NONE) {
1630  next_frame->format = pCodecCtx->sw_pix_fmt;
1631  }
1632  if (next_frame->width <= 0) {
1633  next_frame->width = next_frame2->width;
1634  }
1635  if (next_frame->height <= 0) {
1636  next_frame->height = next_frame2->height;
1637  }
1638  decoded_frame = next_frame;
1639  } else {
1640  // Some hardware decoders can still return software-readable frames.
1641  decoded_frame = next_frame2;
1642  }
1643  }
1644  else
1645 #endif // USE_HW_ACCEL
1646  { // No hardware acceleration used -> no copy from GPU memory needed
1647  decoded_frame = next_frame2;
1648  }
1649 
1650  if (!decoded_frame->data[0]) {
1651  ZmqLogger::Instance()->AppendDebugMethod(
1652  "FFmpegReader::GetAVFrame (Decoded frame missing image data)",
1653  "format", decoded_frame->format,
1654  "width", decoded_frame->width,
1655  "height", decoded_frame->height);
1656  note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty");
1657  break;
1658  }
1659 
1660  // TODO also handle possible further frames
1661  // Use only the first frame like avcodec_decode_video2
1662  frameFinished = 1;
1663  hw_decode_error_count = 0;
1664 #if USE_HW_ACCEL
1665  if (hw_de_on && hw_de_supported && !force_sw_decode) {
1666  hw_decode_succeeded = true;
1667  }
1668 #endif
1669  packet_status.video_decoded++;
1670 
1671  // Allocate image (align 32 for simd)
1672  AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
1673  if (decoded_pix_fmt == AV_PIX_FMT_NONE)
1674  decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
1675  if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) {
1676  throw OutOfMemory("Failed to allocate image buffer", path);
1677  }
1678  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize,
1679  decoded_pix_fmt, info.width, info.height);
1680  pFrame->format = decoded_pix_fmt;
1681  pFrame->width = info.width;
1682  pFrame->height = info.height;
1683  pFrame->color_range = decoded_frame->color_range;
1684  pFrame->colorspace = decoded_frame->colorspace;
1685  pFrame->color_primaries = decoded_frame->color_primaries;
1686  pFrame->color_trc = decoded_frame->color_trc;
1687  pFrame->chroma_location = decoded_frame->chroma_location;
1688 
1689  // Get display PTS from video frame, often different than packet->pts.
1690  // Sending packets to the decoder (i.e. packet->pts) is async,
1691  // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1692  // sending and retrieving are separated by multiple calls to this method.
1693  if (decoded_frame->pts != AV_NOPTS_VALUE) {
1694  // This is the current decoded frame (and should be the pts used) for
1695  // processing this data
1696  video_pts = decoded_frame->pts;
1697  } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) {
1698  // Some videos only set this timestamp (fallback)
1699  video_pts = decoded_frame->pkt_dts;
1700  }
1701 
1702  ZmqLogger::Instance()->AppendDebugMethod(
1703  "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1704 
1705  // break out of loop after each successful image returned
1706  break;
1707  }
1708 #if USE_HW_ACCEL
1709  if (hw_de_on && hw_de_supported && next_frame2 != next_frame) {
1710  AV_FREE_FRAME(&next_frame2);
1711  }
1712  #endif // USE_HW_ACCEL
1713 #else
1714  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1715 
1716  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1717  pFrame = AV_ALLOCATE_FRAME();
1718 
1719  // is frame finished
1720  if (frameFinished) {
1721  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1722  // must make a copy of the image data before this method is called again.
1723  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1724  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1725  info.height);
1726  }
1727 #endif // IS_FFMPEG_3_2
1728 
1729  // deallocate the frame
1730  AV_FREE_FRAME(&next_frame);
1731 
1732  // Did we get a video frame?
1733  return frameFinished;
1734 }
1735 
1736 bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) {
1737 #if USE_HW_ACCEL
1738  if (!hw_decode_failed || force_sw_decode) {
1739  return false;
1740  }
1741 
1742  ZmqLogger::Instance()->AppendDebugMethod(
1743  "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)",
1744  "requested_frame", requested_frame,
1745  "video_packets_read", packet_status.video_read,
1746  "video_packets_decoded", packet_status.video_decoded,
1747  "hw_decode_error_count", hw_decode_error_count);
1748 
1749  force_sw_decode = true;
1750  hw_decode_failed = false;
1751  hw_decode_error_count = 0;
1752 
1753  Close();
1754  Open();
1755  Seek(requested_frame);
1756  return true;
1757 #else
1758  (void) requested_frame;
1759  return false;
1760 #endif
1761 }
1762 
1763 bool FFmpegReader::HardwareDecodeSuccessful() const {
1764 #if USE_HW_ACCEL
1765  return hw_decode_succeeded;
1766 #else
1767  return false;
1768 #endif
1769 }
1770 
1771 // Check the current seek position and determine if we need to seek again
1772 bool FFmpegReader::CheckSeek() {
1773  // Are we seeking for a specific frame?
1774  if (is_seeking) {
1775  const int64_t kSeekRetryMax = 5;
1776  const int kSeekStagnantMax = 2;
1777 
1778  // Determine if both an audio and video packet have been decoded since the seek happened.
1779  // If not, allow the ReadStream method to keep looping
1780  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1781  return false;
1782 
1783  // Check for both streams
1784  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1785  return false;
1786 
1787  // Determine max seeked frame
1788  int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1789  // Track stagnant seek results (no progress between retries)
1790  if (max_seeked_frame == last_seek_max_frame) {
1791  seek_stagnant_count++;
1792  } else {
1793  last_seek_max_frame = max_seeked_frame;
1794  seek_stagnant_count = 0;
1795  }
1796 
1797  // determine if we are "before" the requested frame
1798  if (max_seeked_frame >= seeking_frame) {
1799  // SEEKED TOO FAR
1800  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1801  "is_video_seek", is_video_seek,
1802  "max_seeked_frame", max_seeked_frame,
1803  "seeking_frame", seeking_frame,
1804  "seeking_pts", seeking_pts,
1805  "seek_video_frame_found", seek_video_frame_found,
1806  "seek_audio_frame_found", seek_audio_frame_found);
1807 
1808  // Seek again... to the nearest Keyframe
1809  if (seek_count < kSeekRetryMax) {
1810  Seek(seeking_frame - (10 * seek_count * seek_count));
1811  } else if (seek_stagnant_count >= kSeekStagnantMax) {
1812  // Stagnant seek: force a much earlier target and keep seeking.
1813  Seek(seeking_frame - (10 * kSeekRetryMax * kSeekRetryMax));
1814  } else {
1815  // Retry budget exhausted: keep seeking from a conservative offset.
1816  Seek(seeking_frame - (10 * seek_count * seek_count));
1817  }
1818  } else {
1819  // SEEK WORKED
1820  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1821  "is_video_seek", is_video_seek,
1822  "packet->pts", GetPacketPTS(),
1823  "seeking_pts", seeking_pts,
1824  "seeking_frame", seeking_frame,
1825  "seek_video_frame_found", seek_video_frame_found,
1826  "seek_audio_frame_found", seek_audio_frame_found);
1827 
1828  // Seek worked, and we are "before" the requested frame
1829  is_seeking = false;
1830  seeking_frame = 0;
1831  seeking_pts = -1;
1832  }
1833  }
1834 
1835  // return the pts to seek to (if any)
1836  return is_seeking;
1837 }
1838 
1839 // Process a video packet
1840 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1841  // Get the AVFrame from the current packet
1842  // This sets the video_pts to the correct timestamp
1843  int frame_finished = GetAVFrame();
1844 
1845  // Check if the AVFrame is finished and set it
1846  if (!frame_finished) {
1847  // No AVFrame decoded yet, bail out
1848  if (pFrame) {
1849  RemoveAVFrame(pFrame);
1850  }
1851  return;
1852  }
1853 
1854  // Calculate current frame #
1855  int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1856 
1857  // Track 1st video packet after a successful seek
1858  if (!seek_video_frame_found && is_seeking)
1859  seek_video_frame_found = current_frame;
1860 
1861  // Create or get the existing frame object. Requested frame needs to be created
1862  // in working_cache at least once. Seek can clear the working_cache, so we must
1863  // add the requested frame back to the working_cache here. If it already exists,
1864  // it will be moved to the top of the working_cache.
1865  working_cache.Add(CreateFrame(requested_frame));
1866 
1867  // Debug output
1868  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1869 
1870  // Init some things local (for OpenMP)
1871  AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE)
1872  ? static_cast<AVPixelFormat>(pFrame->format)
1873  : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1874  bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG);
1875  AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range);
1876  int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width;
1877  int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height;
1878  int height = src_height;
1879  int width = src_width;
1880  // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it)
1881  AVFrame *pFrameRGB = pFrameRGB_cached;
1882  if (!pFrameRGB) {
1883  pFrameRGB = AV_ALLOCATE_FRAME();
1884  if (pFrameRGB == nullptr)
1885  throw OutOfMemory("Failed to allocate frame buffer", path);
1886  pFrameRGB_cached = pFrameRGB;
1887  }
1888  AV_RESET_FRAME(pFrameRGB);
1889  uint8_t *buffer = nullptr;
1890 
1891  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1892  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1893  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1894  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1895  // the future.
1896  int max_width = info.width;
1897  int max_height = info.height;
1898 
1899  Clip *parent = static_cast<Clip *>(ParentClip());
1900  if (parent) {
1901  if (parent->ParentTimeline()) {
1902  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1903  max_width = parent->ParentTimeline()->preview_width;
1904  max_height = parent->ParentTimeline()->preview_height;
1905  }
1906  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1907  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1908  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1909  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1910  max_width = std::max(float(max_width), max_width * max_scale_x);
1911  max_height = std::max(float(max_height), max_height * max_scale_y);
1912 
1913  } else if (parent->scale == SCALE_CROP) {
1914  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1915  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1916  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1917  QSize width_size(max_width * max_scale_x,
1918  round(max_width / (float(info.width) / float(info.height))));
1919  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1920  max_height * max_scale_y);
1921  // respect aspect ratio
1922  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1923  max_width = std::max(max_width, width_size.width());
1924  max_height = std::max(max_height, width_size.height());
1925  } else {
1926  max_width = std::max(max_width, height_size.width());
1927  max_height = std::max(max_height, height_size.height());
1928  }
1929 
1930  } else {
1931  // Scale video to equivalent unscaled size
1932  // Since the preview window can change sizes, we want to always
1933  // scale against the ratio of original video size to timeline size
1934  float preview_ratio = 1.0;
1935  if (parent->ParentTimeline()) {
1936  Timeline *t = (Timeline *) parent->ParentTimeline();
1937  preview_ratio = t->preview_width / float(t->info.width);
1938  }
1939  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1940  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1941  max_width = info.width * max_scale_x * preview_ratio;
1942  max_height = info.height * max_scale_y * preview_ratio;
1943  }
1944 
1945  // If a crop effect is resizing the image, request enough pixels to preserve detail
1946  ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height);
1947  }
1948 
1949  if (HasMaxDecodeSize()) {
1950  QSize bounded_size(max_width, max_height);
1951  const QSize max_decode_size(MaxDecodeWidth(), MaxDecodeHeight());
1952  if (bounded_size.width() > max_decode_size.width() ||
1953  bounded_size.height() > max_decode_size.height()) {
1954  bounded_size.scale(max_decode_size, Qt::KeepAspectRatio);
1955  max_width = bounded_size.width();
1956  max_height = bounded_size.height();
1957  }
1958  }
1959 
1960  // Determine if image needs to be scaled (for performance reasons)
1961  int original_height = src_height;
1962  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1963  // Override width and height (but maintain aspect ratio)
1964  float ratio = float(width) / float(height);
1965  int possible_width = round(max_height * ratio);
1966  int possible_height = round(max_width / ratio);
1967 
1968  if (possible_width <= max_width) {
1969  // use calculated width, and max_height
1970  width = possible_width;
1971  height = max_height;
1972  } else {
1973  // use max_width, and calculated height
1974  width = max_width;
1975  height = possible_height;
1976  }
1977  }
1978 
1979  // Determine required buffer size and allocate buffer
1980  const int bytes_per_pixel = 4;
1981  int raw_buffer_size = (width * height * bytes_per_pixel) + 128;
1982 
1983  // Aligned memory allocation (for speed)
1984  constexpr size_t ALIGNMENT = 32; // AVX2
1985  int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
1986  buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT);
1987 
1988  // Copy picture data from one AVFrame (or AVPicture) to another one.
1989  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1990 
1991  int scale_mode = SWS_FAST_BILINEAR;
1992  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1993  scale_mode = SWS_BICUBIC;
1994  }
1995  img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1996  if (!img_convert_ctx)
1997  throw OutOfMemory("Failed to initialize sws context", path);
1998  const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
1999  const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
2000  const int dst_full_range = 1; // RGB outputs are full-range
2001  sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0,
2002  dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16);
2003 
2004  if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) {
2005 #if USE_HW_ACCEL
2006  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2007  hw_decode_failed = true;
2008  ZmqLogger::Instance()->AppendDebugMethod(
2009  "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)",
2010  "requested_frame", requested_frame,
2011  "current_frame", current_frame,
2012  "src_pix_fmt", src_pix_fmt,
2013  "src_width", src_width,
2014  "src_height", src_height);
2015  }
2016 #endif
2017  if (pFrame) {
2018  RemoveAVFrame(pFrame);
2019  pFrame = NULL;
2020  }
2021  return;
2022  }
2023 
2024  // Resize / Convert to RGB
2025  const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
2026  original_height, pFrameRGB->data, pFrameRGB->linesize);
2027  if (scaled_lines <= 0) {
2028 #if USE_HW_ACCEL
2029  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2030  hw_decode_failed = true;
2031  ZmqLogger::Instance()->AppendDebugMethod(
2032  "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)",
2033  "requested_frame", requested_frame,
2034  "current_frame", current_frame,
2035  "scaled_lines", scaled_lines,
2036  "src_pix_fmt", src_pix_fmt,
2037  "src_width", src_width,
2038  "src_height", src_height);
2039  }
2040 #endif
2041  free(buffer);
2042  AV_RESET_FRAME(pFrameRGB);
2043  RemoveAVFrame(pFrame);
2044  pFrame = NULL;
2045  return;
2046  }
2047 
2048  // Create or get the existing frame object
2049  std::shared_ptr<Frame> f = CreateFrame(current_frame);
2050 
2051  // Add Image data to frame
2052  if (!ffmpeg_has_alpha(src_pix_fmt)) {
2053  // Add image with no alpha channel, Speed optimization
2054  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
2055  } else {
2056  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
2057  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
2058  }
2059 
2060  // Update working cache
2061  working_cache.Add(f);
2062 
2063  // Keep track of last last_video_frame
2064  last_video_frame = f;
2065 
2066  // Free the RGB image
2067  AV_RESET_FRAME(pFrameRGB);
2068 
2069  // Remove frame and packet
2070  RemoveAVFrame(pFrame);
2071 
2072  // Get video PTS in seconds
2073  video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2074 
2075  // Debug output
2076  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
2077 }
2078 
2079 // Process an audio packet
2080 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
2081  AudioLocation location;
2082  // Calculate location of current audio packet
2083  if (packet && packet->pts != AV_NOPTS_VALUE) {
2084  // Determine related video frame and starting sample # from audio PTS
2085  location = GetAudioPTSLocation(packet->pts);
2086 
2087  // Track 1st audio packet after a successful seek
2088  if (!seek_audio_frame_found && is_seeking)
2089  seek_audio_frame_found = location.frame;
2090  }
2091 
2092  // Create or get the existing frame object. Requested frame needs to be created
2093  // in working_cache at least once. Seek can clear the working_cache, so we must
2094  // add the requested frame back to the working_cache here. If it already exists,
2095  // it will be moved to the top of the working_cache.
2096  working_cache.Add(CreateFrame(requested_frame));
2097 
2098  // Debug output
2099  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
2100  "requested_frame", requested_frame,
2101  "target_frame", location.frame,
2102  "starting_sample", location.sample_start);
2103 
2104  // Init an AVFrame to hold the decoded audio samples
2105  int frame_finished = 0;
2106  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
2107  AV_RESET_FRAME(audio_frame);
2108 
2109  int packet_samples = 0;
2110  int data_size = 0;
2111 
2112 #if IS_FFMPEG_3_2
2113  int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
2114  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
2115  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
2116  }
2117  else {
2118  int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
2119  if (receive_frame_err >= 0) {
2120  frame_finished = 1;
2121  }
2122  if (receive_frame_err == AVERROR_EOF) {
2123  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
2124  packet_status.audio_eof = true;
2125  }
2126  if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
2127  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
2128  avcodec_flush_buffers(aCodecCtx);
2129  }
2130  if (receive_frame_err != 0) {
2131  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
2132  }
2133  }
2134 #else
2135  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
2136 #endif
2137 
2138  if (frame_finished) {
2139  packet_status.audio_decoded++;
2140 
2141  // This can be different than the current packet, so we need to look
2142  // at the current AVFrame from the audio decoder. This timestamp should
2143  // be used for the remainder of this function
2144  audio_pts = audio_frame->pts;
2145 
2146  // Determine related video frame and starting sample # from audio PTS
2147  location = GetAudioPTSLocation(audio_pts);
2148 
2149  // determine how many samples were decoded
2150  int plane_size = -1;
2151 #if HAVE_CH_LAYOUT
2152  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
2153 #else
2154  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
2155 #endif
2156  data_size = av_samples_get_buffer_size(&plane_size, nb_channels,
2157  audio_frame->nb_samples, (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
2158 
2159  // Calculate total number of samples
2160  packet_samples = audio_frame->nb_samples * nb_channels;
2161  } else {
2162  if (audio_frame) {
2163  // Free audio frame
2164  AV_FREE_FRAME(&audio_frame);
2165  }
2166  }
2167 
2168  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
2169  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
2170 
2171  // Bail if no samples found
2172  if (pts_remaining_samples == 0) {
2173  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
2174  "packet_samples", packet_samples,
2175  "info.channels", info.channels,
2176  "pts_remaining_samples", pts_remaining_samples);
2177  return;
2178  }
2179 
2180  while (pts_remaining_samples) {
2181  // Get Samples per frame (for this frame number)
2182  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
2183 
2184  // Calculate # of samples to add to this frame
2185  int samples = samples_per_frame - previous_packet_location.sample_start;
2186  if (samples > pts_remaining_samples)
2187  samples = pts_remaining_samples;
2188 
2189  // Decrement remaining samples
2190  pts_remaining_samples -= samples;
2191 
2192  if (pts_remaining_samples > 0) {
2193  // next frame
2194  previous_packet_location.frame++;
2195  previous_packet_location.sample_start = 0;
2196  } else {
2197  // Increment sample start
2198  previous_packet_location.sample_start += samples;
2199  }
2200  }
2201 
2202  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
2203  "packet_samples", packet_samples,
2204  "info.channels", info.channels,
2205  "info.sample_rate", info.sample_rate,
2206  "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx));
2207 
2208  // Create output frame
2209  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
2210  AV_RESET_FRAME(audio_converted);
2211  audio_converted->nb_samples = audio_frame->nb_samples;
2212  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_FLTP, 0);
2213 
2214  SWRCONTEXT *avr = avr_ctx;
2215  // setup resample context if needed
2216  if (!avr) {
2217  avr = SWR_ALLOC();
2218 #if HAVE_CH_LAYOUT
2219  AVChannelLayout input_layout = ffmpeg_get_valid_channel_layout(
2220  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, info.channels);
2221  AVChannelLayout output_layout = ffmpeg_get_valid_channel_layout(
2222  input_layout, info.channels);
2223  int in_layout_err = av_opt_set_chlayout(avr, "in_chlayout", &input_layout, 0);
2224  int out_layout_err = av_opt_set_chlayout(avr, "out_chlayout", &output_layout, 0);
2225 #else
2226  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2227  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2228  av_opt_set_int(avr, "in_channels", info.channels, 0);
2229  av_opt_set_int(avr, "out_channels", info.channels, 0);
2230 #endif
2231  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
2232  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
2233  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
2234  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
2235  int swr_init_err = SWR_INIT(avr);
2236 #if HAVE_CH_LAYOUT
2237  av_channel_layout_uninit(&input_layout);
2238  av_channel_layout_uninit(&output_layout);
2239  if (in_layout_err < 0 || out_layout_err < 0 || swr_init_err < 0) {
2240  SWR_FREE(&avr);
2241  throw InvalidChannels("Could not initialize FFmpeg audio channel layout or resampler.", path);
2242  }
2243 #else
2244  if (swr_init_err < 0) {
2245  SWR_FREE(&avr);
2246  throw InvalidChannels("Could not initialize FFmpeg audio resampler.", path);
2247  }
2248 #endif
2249  avr_ctx = avr;
2250  }
2251 
2252  // Convert audio samples
2253  int nb_samples = SWR_CONVERT(avr, // audio resample context
2254  audio_converted->data, // output data pointers
2255  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
2256  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
2257  audio_frame->data, // input data pointers
2258  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
2259  audio_frame->nb_samples); // number of input samples to convert
2260 
2261 
2262  int64_t starting_frame_number = -1;
2263  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
2264  // Array of floats (to hold samples for each channel)
2265  starting_frame_number = location.frame;
2266  int channel_buffer_size = nb_samples;
2267  auto *channel_buffer = (float *) (audio_converted->data[channel_filter]);
2268 
2269  // Loop through samples, and add them to the correct frames
2270  int start = location.sample_start;
2271  int remaining_samples = channel_buffer_size;
2272  while (remaining_samples > 0) {
2273  // Get Samples per frame (for this frame number)
2274  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
2275 
2276  // Calculate # of samples to add to this frame
2277  int samples = std::fmin(samples_per_frame - start, remaining_samples);
2278 
2279  // Create or get the existing frame object
2280  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
2281 
2282  // Add samples for current channel to the frame.
2283  f->AddAudio(true, channel_filter, start, channel_buffer, samples, 1.0f);
2284 
2285  // Debug output
2286  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
2287  "frame", starting_frame_number,
2288  "start", start,
2289  "samples", samples,
2290  "channel", channel_filter,
2291  "samples_per_frame", samples_per_frame);
2292 
2293  // Add or update cache
2294  working_cache.Add(f);
2295 
2296  // Decrement remaining samples
2297  remaining_samples -= samples;
2298 
2299  // Increment buffer (to next set of samples)
2300  if (remaining_samples > 0)
2301  channel_buffer += samples;
2302 
2303  // Increment frame number
2304  starting_frame_number++;
2305 
2306  // Reset starting sample #
2307  start = 0;
2308  }
2309  }
2310 
2311  // Free AVFrames
2312  av_free(audio_converted->data[0]);
2313  AV_FREE_FRAME(&audio_converted);
2314  AV_FREE_FRAME(&audio_frame);
2315 
2316  // Get audio PTS in seconds
2317  audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2318 
2319  // Debug output
2320  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
2321  "requested_frame", requested_frame,
2322  "starting_frame", location.frame,
2323  "end_frame", starting_frame_number - 1,
2324  "audio_pts_seconds", audio_pts_seconds);
2325 
2326 }
2327 
2328 
2329 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
2330 void FFmpegReader::Seek(int64_t requested_frame) {
2331  // Adjust for a requested frame that is too small or too large
2332  if (requested_frame < 1)
2333  requested_frame = 1;
2334  if (requested_frame > info.video_length)
2335  requested_frame = info.video_length;
2336  if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
2337  // Not possible to search past largest_frame once EOF is reached (no more packets)
2338  return;
2339  }
2340 
2341  // Debug output
2342  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
2343  "requested_frame", requested_frame,
2344  "seek_count", seek_count,
2345  "last_frame", last_frame);
2346 
2347  // Clear working cache (since we are seeking to another location in the file)
2348  working_cache.Clear();
2349 
2350  // Reset the last frame variable
2351  video_pts = 0.0;
2352  video_pts_seconds = NO_PTS_OFFSET;
2353  audio_pts = 0.0;
2354  audio_pts_seconds = NO_PTS_OFFSET;
2355  hold_packet = false;
2356  last_frame = 0;
2357  current_video_frame = 0;
2358  largest_frame_processed = 0;
2359  last_final_video_frame.reset();
2360  bool has_audio_override = info.has_audio;
2361  bool has_video_override = info.has_video;
2362 
2363  // Init end-of-file detection variables
2364  packet_status.reset(false);
2365 
2366  // Increment seek count
2367  seek_count++;
2368 
2369  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
2370  int buffer_amount = 12;
2371  if (requested_frame - buffer_amount < 20) {
2372  // prevent Open() from seeking again
2373  is_seeking = true;
2374 
2375  // Close and re-open file (basically seeking to frame 1)
2376  Close();
2377  Open();
2378 
2379  // Update overrides (since closing and re-opening might update these)
2380  info.has_audio = has_audio_override;
2381  info.has_video = has_video_override;
2382 
2383  // Not actually seeking, so clear these flags
2384  is_seeking = false;
2385  if (seek_count == 1) {
2386  // Don't redefine this on multiple seek attempts for a specific frame
2387  seeking_frame = 1;
2388  seeking_pts = ConvertFrameToVideoPTS(1);
2389  }
2390  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2391  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2392 
2393  } else {
2394  // Seek to nearest key-frame (aka, i-frame)
2395  bool seek_worked = false;
2396  int64_t seek_target = 0;
2397 
2398  // Seek video stream (if any), except album arts
2399  if (!seek_worked && info.has_video && !HasAlbumArt()) {
2400  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
2401  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2402  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking video stream");
2403  } else {
2404  // VIDEO SEEK
2405  is_video_seek = true;
2406  seek_worked = true;
2407  }
2408  }
2409 
2410  // Seek audio stream (if not already seeked... and if an audio stream is found)
2411  if (!seek_worked && info.has_audio) {
2412  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
2413  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2414  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking audio stream");
2415  } else {
2416  // AUDIO SEEK
2417  is_video_seek = false;
2418  seek_worked = true;
2419  }
2420  }
2421 
2422  // Was the seek successful?
2423  if (seek_worked) {
2424  // Flush audio buffer
2425  if (info.has_audio)
2426  avcodec_flush_buffers(aCodecCtx);
2427 
2428  // Flush video buffer
2429  if (info.has_video)
2430  avcodec_flush_buffers(pCodecCtx);
2431 
2432  // Reset previous audio location to zero
2433  previous_packet_location.frame = -1;
2434  previous_packet_location.sample_start = 0;
2435 
2436  // init seek flags
2437  is_seeking = true;
2438  if (seek_count == 1) {
2439  // Don't redefine this on multiple seek attempts for a specific frame
2440  seeking_pts = seek_target;
2441  seeking_frame = requested_frame;
2442  }
2443  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2444  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2445 
2446  } else {
2447  // seek failed
2448  seeking_pts = 0;
2449  seeking_frame = 0;
2450 
2451  // prevent Open() from seeking again
2452  is_seeking = true;
2453 
2454  // Close and re-open file (basically seeking to frame 1)
2455  Close();
2456  Open();
2457 
2458  // Not actually seeking, so clear these flags
2459  is_seeking = false;
2460 
2461  // disable seeking for this reader (since it failed)
2462  enable_seek = false;
2463 
2464  // Update overrides (since closing and re-opening might update these)
2465  info.has_audio = has_audio_override;
2466  info.has_video = has_video_override;
2467  }
2468  }
2469 }
2470 
2471 // Get the PTS for the current video packet
2472 int64_t FFmpegReader::GetPacketPTS() {
2473  if (packet) {
2474  int64_t current_pts = packet->pts;
2475  if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
2476  current_pts = packet->dts;
2477 
2478  // Return adjusted PTS
2479  return current_pts;
2480  } else {
2481  // No packet, return NO PTS
2482  return AV_NOPTS_VALUE;
2483  }
2484 }
2485 
2486 // Update PTS Offset (if any)
2487 void FFmpegReader::UpdatePTSOffset() {
2488  if (pts_offset_seconds != NO_PTS_OFFSET) {
2489  // Skip this method if we have already set PTS offset
2490  return;
2491  }
2492  pts_offset_seconds = 0.0;
2493  double video_pts_offset_seconds = 0.0;
2494  double audio_pts_offset_seconds = 0.0;
2495 
2496  bool has_video_pts = false;
2497  if (!info.has_video) {
2498  // Mark as checked
2499  has_video_pts = true;
2500  }
2501  bool has_audio_pts = false;
2502  if (!info.has_audio) {
2503  // Mark as checked
2504  has_audio_pts = true;
2505  }
2506 
2507  // Loop through the stream (until a packet from all streams is found)
2508  while (!has_video_pts || !has_audio_pts) {
2509  // Get the next packet (if any)
2510  if (GetNextPacket() < 0)
2511  // Break loop when no more packets found
2512  break;
2513 
2514  // Get PTS of this packet
2515  int64_t pts = GetPacketPTS();
2516 
2517  // Video packet
2518  if (!has_video_pts && packet->stream_index == videoStream) {
2519  // Get the video packet start time (in seconds)
2520  video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble());
2521 
2522  // Is timestamp close to zero (within X seconds)
2523  // Ignore wildly invalid timestamps (i.e. -234923423423)
2524  if (std::abs(video_pts_offset_seconds) <= 10.0) {
2525  has_video_pts = true;
2526  }
2527  }
2528  else if (!has_audio_pts && packet->stream_index == audioStream) {
2529  // Get the audio packet start time (in seconds)
2530  audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
2531 
2532  // Is timestamp close to zero (within X seconds)
2533  // Ignore wildly invalid timestamps (i.e. -234923423423)
2534  if (std::abs(audio_pts_offset_seconds) <= 10.0) {
2535  has_audio_pts = true;
2536  }
2537  }
2538  }
2539 
2540  // Choose timestamp origin:
2541  // - If video exists, anchor timeline frame mapping to video start.
2542  // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2.
2543  // - If no video exists (audio-only readers), use audio start.
2544  if (info.has_video && has_video_pts) {
2545  pts_offset_seconds = video_pts_offset_seconds;
2546  } else if (!info.has_video && has_audio_pts) {
2547  pts_offset_seconds = audio_pts_offset_seconds;
2548  } else if (has_video_pts && has_audio_pts) {
2549  // Fallback when stream flags are unusual but both timestamps exist.
2550  pts_offset_seconds = video_pts_offset_seconds;
2551  }
2552 }
2553 
2554 // Convert PTS into Frame Number
2555 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
2556  // Apply PTS offset
2557  int64_t previous_video_frame = current_video_frame;
2558  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2559  const double video_timebase_value =
2560  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2561  ? info.video_timebase.ToDouble()
2562  : (1.0 / 30.0);
2563 
2564  // Get the video packet start time (in seconds)
2565  double video_seconds = (double(pts) * video_timebase_value) + pts_offset_seconds;
2566 
2567  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2568  int64_t frame = round(video_seconds * fps_value) + 1;
2569 
2570  // Keep track of the expected video frame #
2571  if (current_video_frame == 0)
2572  current_video_frame = frame;
2573  else {
2574 
2575  // Sometimes frames are duplicated due to identical (or similar) timestamps
2576  if (frame == previous_video_frame) {
2577  // return -1 frame number
2578  frame = -1;
2579  } else {
2580  // Increment expected frame
2581  current_video_frame++;
2582  }
2583  }
2584 
2585  // Return frame #
2586  return frame;
2587 }
2588 
2589 // Convert Frame Number into Video PTS
2590 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2591  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2592  const double video_timebase_value =
2593  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2594  ? info.video_timebase.ToDouble()
2595  : (1.0 / 30.0);
2596 
2597  // Get timestamp of this frame (in seconds)
2598  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2599 
2600  // Calculate the # of video packets in this timestamp
2601  int64_t video_pts = round(seconds / video_timebase_value);
2602 
2603  // Apply PTS offset (opposite)
2604  return video_pts;
2605 }
2606 
2607 // Convert Frame Number into Video PTS
2608 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2609  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2610  const double audio_timebase_value =
2611  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2612  ? info.audio_timebase.ToDouble()
2613  : (1.0 / 48000.0);
2614 
2615  // Get timestamp of this frame (in seconds)
2616  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2617 
2618  // Calculate the # of audio packets in this timestamp
2619  int64_t audio_pts = round(seconds / audio_timebase_value);
2620 
2621  // Apply PTS offset (opposite)
2622  return audio_pts;
2623 }
2624 
2625 // Calculate Starting video frame and sample # for an audio PTS
2626 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2627  const double audio_timebase_value =
2628  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2629  ? info.audio_timebase.ToDouble()
2630  : (1.0 / 48000.0);
2631  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2632 
2633  // Get the audio packet start time (in seconds)
2634  double audio_seconds = (double(pts) * audio_timebase_value) + pts_offset_seconds;
2635 
2636  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2637  double frame = (audio_seconds * fps_value) + 1;
2638 
2639  // Frame # as a whole number (no more decimals)
2640  int64_t whole_frame = int64_t(frame);
2641 
2642  // Remove the whole number, and only get the decimal of the frame
2643  double sample_start_percentage = frame - double(whole_frame);
2644 
2645  // Get Samples per frame
2646  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2647 
2648  // Calculate the sample # to start on
2649  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2650 
2651  // Protect against broken (i.e. negative) timestamps
2652  if (whole_frame < 1)
2653  whole_frame = 1;
2654  if (sample_start < 0)
2655  sample_start = 0;
2656 
2657  // Prepare final audio packet location
2658  AudioLocation location = {whole_frame, sample_start};
2659 
2660  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2661  if (previous_packet_location.frame != -1) {
2662  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2663  int64_t orig_frame = location.frame;
2664  int orig_start = location.sample_start;
2665 
2666  // Update sample start, to prevent gaps in audio
2667  location.sample_start = previous_packet_location.sample_start;
2668  location.frame = previous_packet_location.frame;
2669 
2670  // Debug output
2671  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2672 
2673  } else {
2674  // Debug output
2675  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2676  }
2677  }
2678 
2679  // Set previous location
2680  previous_packet_location = location;
2681 
2682  // Return the associated video frame and starting sample #
2683  return location;
2684 }
2685 
2686 // Create a new Frame (or return an existing one) and add it to the working queue.
2687 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2688  // Check working cache
2689  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2690 
2691  if (!output) {
2692  // (re-)Check working cache
2693  output = working_cache.GetFrame(requested_frame);
2694  if(output) return output;
2695 
2696  // Create a new frame on the working cache
2697  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2698  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2699  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2700  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2701 
2702  working_cache.Add(output);
2703 
2704  // Set the largest processed frame (if this is larger)
2705  if (requested_frame > largest_frame_processed)
2706  largest_frame_processed = requested_frame;
2707  }
2708  // Return frame
2709  return output;
2710 }
2711 
2712 // Determine if frame is partial due to seek
2713 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2714 
2715  // Sometimes a seek gets partial frames, and we need to remove them
2716  bool seek_trash = false;
2717  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2718  if (seek_video_frame_found > max_seeked_frame) {
2719  max_seeked_frame = seek_video_frame_found;
2720  }
2721  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2722  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2723  seek_trash = true;
2724  }
2725 
2726  return seek_trash;
2727 }
2728 
2729 // Check the working queue, and move finished frames to the finished queue
2730 void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2731 
2732  // Prevent async calls to the following code
2733  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2734 
2735  // Get a list of current working queue frames in the cache (in-progress frames)
2736  std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2737  std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2738 
2739  // Loop through all working queue frames (sorted by frame #)
2740  for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2741  {
2742  // Get working frame
2743  std::shared_ptr<Frame> f = *working_itr;
2744 
2745  // Was a frame found? Is frame requested yet?
2746  if (!f || f->number > requested_frame) {
2747  // If not, skip to next one
2748  continue;
2749  }
2750 
2751  // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2752  double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2753  double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2754 
2755  // Determine if video and audio are ready (based on timestamps)
2756  bool is_video_ready = false;
2757  bool is_audio_ready = false;
2758  double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2759  if ((frame_pts_seconds <= video_pts_seconds)
2760  || (recent_pts_diff > 1.5)
2761  || packet_status.video_eof || packet_status.end_of_file) {
2762  // Video stream is past this frame (so it must be done)
2763  // OR video stream is too far behind, missing, or end-of-file
2764  is_video_ready = true;
2765  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2766  "frame_number", f->number,
2767  "frame_pts_seconds", frame_pts_seconds,
2768  "video_pts_seconds", video_pts_seconds,
2769  "recent_pts_diff", recent_pts_diff);
2770  if (info.has_video && !f->has_image_data &&
2771  (packet_status.video_eof || packet_status.end_of_file)) {
2772  // Frame has no image data. Prefer timeline-previous frames to preserve
2773  // visual order, especially when decode/prefetch is out-of-order.
2774  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2775  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2776  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2777  }
2778 
2779  // Fall back to last finalized timeline image (survives cache churn).
2780  if (!f->has_image_data
2781  && last_final_video_frame
2782  && last_final_video_frame->has_image_data
2783  && last_final_video_frame->number <= f->number) {
2784  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2785  }
2786 
2787  // Fall back to the last decoded image only when it is not from the future.
2788  if (!f->has_image_data
2789  && last_video_frame
2790  && last_video_frame->has_image_data
2791  && last_video_frame->number <= f->number) {
2792  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2793  }
2794 
2795  // Last-resort fallback if no prior image is available.
2796  if (!f->has_image_data) {
2797  ZmqLogger::Instance()->AppendDebugMethod(
2798  "FFmpegReader::CheckWorkingFrames (no previous image found; using black frame)",
2799  "frame_number", f->number);
2800  f->AddColor("#000000");
2801  }
2802  }
2803  }
2804 
2805  double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2806  if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2807  || (recent_pts_diff > 1.5)
2808  || packet_status.audio_eof || packet_status.end_of_file) {
2809  // Audio stream is past this frame (so it must be done)
2810  // OR audio stream is too far behind, missing, or end-of-file
2811  // Adding a bit of margin here, to allow for partial audio packets
2812  is_audio_ready = true;
2813  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2814  "frame_number", f->number,
2815  "frame_pts_seconds", frame_pts_seconds,
2816  "audio_pts_seconds", audio_pts_seconds,
2817  "audio_pts_diff", audio_pts_diff,
2818  "recent_pts_diff", recent_pts_diff);
2819  }
2820  bool is_seek_trash = IsPartialFrame(f->number);
2821 
2822  // Adjust for available streams
2823  if (!info.has_video) is_video_ready = true;
2824  if (!info.has_audio) is_audio_ready = true;
2825 
2826  // Debug output
2827  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2828  "frame_number", f->number,
2829  "is_video_ready", is_video_ready,
2830  "is_audio_ready", is_audio_ready,
2831  "video_eof", packet_status.video_eof,
2832  "audio_eof", packet_status.audio_eof,
2833  "end_of_file", packet_status.end_of_file);
2834 
2835  // Check if working frame is final
2836  if (info.has_video && !f->has_image_data
2837  && !packet_status.end_of_file && !is_seek_trash) {
2838  if (info.has_single_image) {
2839  // For still-image video (including attached cover art), reuse the most
2840  // recent image so playback does not stall waiting for video EOF.
2841  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2842  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2843  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2844  }
2845  if (!f->has_image_data
2846  && last_final_video_frame
2847  && last_final_video_frame->has_image_data
2848  && last_final_video_frame->number <= f->number) {
2849  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2850  }
2851  if (!f->has_image_data
2852  && last_video_frame
2853  && last_video_frame->has_image_data
2854  && last_video_frame->number <= f->number) {
2855  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2856  }
2857  }
2858 
2859  // If both streams have advanced past this frame but the decoder never
2860  // produced image data for it, reuse the most recent non-future image.
2861  // This avoids stalling indefinitely on sparse/missing decoded frames.
2862  if (!f->has_image_data && is_video_ready && is_audio_ready) {
2863  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2864  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2865  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2866  }
2867  if (!f->has_image_data
2868  && last_final_video_frame
2869  && last_final_video_frame->has_image_data
2870  && last_final_video_frame->number <= f->number) {
2871  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2872  }
2873  if (!f->has_image_data
2874  && last_video_frame
2875  && last_video_frame->has_image_data
2876  && last_video_frame->number <= f->number) {
2877  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2878  }
2879  }
2880 
2881  // Do not finalize non-EOF video frames without decoded image data.
2882  // This prevents repeated previous-frame fallbacks being cached as real frames.
2883  if (!f->has_image_data) {
2884  continue;
2885  }
2886  }
2887  if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2888  // Debug output
2889  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2890  "requested_frame", requested_frame,
2891  "f->number", f->number,
2892  "is_seek_trash", is_seek_trash,
2893  "Working Cache Count", working_cache.Count(),
2894  "Final Cache Count", final_cache.Count(),
2895  "end_of_file", packet_status.end_of_file);
2896 
2897  if (!is_seek_trash) {
2898  // Move frame to final cache
2899  final_cache.Add(f);
2900  if (f->has_image_data) {
2901  last_final_video_frame = f;
2902  }
2903 
2904  // Remove frame from working cache
2905  working_cache.Remove(f->number);
2906 
2907  // Update last frame processed
2908  last_frame = f->number;
2909  } else {
2910  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2911  working_cache.Remove(f->number);
2912  }
2913 
2914  }
2915  }
2916 
2917  // Clear vector of frames
2918  working_frames.clear();
2919  working_frames.shrink_to_fit();
2920 }
2921 
2922 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2923 void FFmpegReader::CheckFPS() {
2924  if (check_fps) {
2925  // Do not check FPS more than 1 time
2926  return;
2927  } else {
2928  check_fps = true;
2929  }
2930 
2931  int frames_per_second[3] = {0,0,0};
2932  int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
2933  int fps_index = 0;
2934 
2935  int all_frames_detected = 0;
2936  int starting_frames_detected = 0;
2937 
2938  // Loop through the stream
2939  while (true) {
2940  // Get the next packet (if any)
2941  if (GetNextPacket() < 0)
2942  // Break loop when no more packets found
2943  break;
2944 
2945  // Video packet
2946  if (packet->stream_index == videoStream) {
2947  // Get the video packet start time (in seconds)
2948  double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2949  fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
2950 
2951  // Is this video packet from the first few seconds?
2952  if (fps_index >= 0 && fps_index < max_fps_index) {
2953  // Yes, keep track of how many frames per second (over the first few seconds)
2954  starting_frames_detected++;
2955  frames_per_second[fps_index]++;
2956  }
2957 
2958  // Track all video packets detected
2959  all_frames_detected++;
2960  }
2961  }
2962 
2963  // Calculate FPS (based on the first few seconds of video packets)
2964  float avg_fps = 30.0;
2965  if (starting_frames_detected > 0 && fps_index > 0) {
2966  avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
2967  }
2968 
2969  // Verify average FPS is a reasonable value
2970  if (avg_fps < 8.0) {
2971  // Invalid FPS assumed, so switching to a sane default FPS instead
2972  avg_fps = 30.0;
2973  }
2974 
2975  // Update FPS (truncate average FPS to Integer)
2976  info.fps = Fraction(int(avg_fps), 1);
2977 
2978  // Update Duration and Length
2979  if (all_frames_detected > 0) {
2980  // Use all video frames detected to calculate # of frames
2981  info.video_length = all_frames_detected;
2982  info.duration = all_frames_detected / avg_fps;
2983  } else {
2984  // Use previous duration to calculate # of frames
2985  info.video_length = info.duration * avg_fps;
2986  }
2987 
2988  // Update video bit rate
2989  info.video_bit_rate = info.file_size / info.duration;
2990 }
2991 
2992 // Remove AVFrame from cache (and deallocate its memory)
2993 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2994  // Remove pFrame (if exists)
2995  if (remove_frame) {
2996  // Free memory
2997  av_freep(&remove_frame->data[0]);
2998 #ifndef WIN32
2999  AV_FREE_FRAME(&remove_frame);
3000 #endif
3001  }
3002 }
3003 
3004 // Remove AVPacket from cache (and deallocate its memory)
3005 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
3006  // deallocate memory for packet
3007  AV_FREE_PACKET(remove_packet);
3008 
3009  // Delete the object
3010  delete remove_packet;
3011 }
3012 
3013 // Generate JSON string of this object
3014 std::string FFmpegReader::Json() const {
3015 
3016  // Return formatted string
3017  return JsonValue().toStyledString();
3018 }
3019 
3020 // Generate Json::Value for this object
3021 Json::Value FFmpegReader::JsonValue() const {
3022 
3023  // Create root json object
3024  Json::Value root = ReaderBase::JsonValue(); // get parent properties
3025  root["type"] = "FFmpegReader";
3026  root["path"] = path;
3027  switch (duration_strategy) {
3028  case DurationStrategy::VideoPreferred:
3029  root["duration_strategy"] = "VideoPreferred";
3030  break;
3031  case DurationStrategy::AudioPreferred:
3032  root["duration_strategy"] = "AudioPreferred";
3033  break;
3034  case DurationStrategy::LongestStream:
3035  default:
3036  root["duration_strategy"] = "LongestStream";
3037  break;
3038  }
3039 
3040  // return JsonValue
3041  return root;
3042 }
3043 
3044 // Load JSON string into this object
3045 void FFmpegReader::SetJson(const std::string value) {
3046 
3047  // Parse JSON string into JSON objects
3048  try {
3049  const Json::Value root = openshot::stringToJson(value);
3050  // Set all values that match
3051  SetJsonValue(root);
3052  }
3053  catch (const std::exception& e) {
3054  // Error parsing JSON (or missing keys)
3055  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
3056  }
3057 }
3058 
3059 // Load Json::Value into this object
3060 void FFmpegReader::SetJsonValue(const Json::Value root) {
3061 
3062  // Set parent data
3063  ReaderBase::SetJsonValue(root);
3064 
3065  // Set data from Json (if key is found)
3066  if (!root["path"].isNull())
3067  path = root["path"].asString();
3068  if (!root["duration_strategy"].isNull()) {
3069  const std::string strategy = root["duration_strategy"].asString();
3070  if (strategy == "VideoPreferred") {
3071  duration_strategy = DurationStrategy::VideoPreferred;
3072  } else if (strategy == "AudioPreferred") {
3073  duration_strategy = DurationStrategy::AudioPreferred;
3074  } else {
3075  duration_strategy = DurationStrategy::LongestStream;
3076  }
3077  }
3078 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::CacheMemory::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheMemory.cpp:224
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:317
openshot::ReaderInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
openshot::FFmpegReader::FFmpegReader
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Definition: FFmpegReader.cpp:102
openshot::Fraction::ToFloat
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
openshot::Settings::HARDWARE_DECODER
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:71
openshot::Coordinate::Y
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
openshot::CacheMemory::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheMemory.cpp:240
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::ReaderBase::JsonValue
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:109
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:178
openshot::TimelineBase::preview_width
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
openshot::PacketStatus::reset
void reset(bool eof)
Definition: FFmpegReader.h:70
openshot::ReaderBase::MaxDecodeHeight
int MaxDecodeHeight() const
Return the current maximum decoded frame height (0 when unlimited).
Definition: ReaderBase.cpp:261
openshot::CacheMemory::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:84
openshot::FFmpegReader::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: FFmpegReader.cpp:1199
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:326
openshot::CacheMemory::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:47
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:309
openshot::ReaderBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:160
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:258
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AnimatedCurve.h:24
openshot::Point::co
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
openshot::Clip::scale_y
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:326
openshot::AudioLocation
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
openshot::AudioLocation::frame
int64_t frame
Definition: AudioLocation.h:26
openshot::ZmqLogger::Log
void Log(std::string message)
Log message to all subscribers of this logger (if any)
Definition: ZmqLogger.cpp:103
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::DurationStrategy::AudioPreferred
@ AudioPreferred
Prefer the audio stream's duration, fallback to video then container.
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
openshot::AudioLocation::sample_start
int sample_start
Definition: AudioLocation.h:27
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:313
MemoryTrim.h
Cross-platform helper to encourage returning freed memory to the OS.
openshot::Keyframe::GetMaxPoint
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
openshot::ReaderBase::info
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:90
openshot::ReaderInfo::interlaced_frame
bool interlaced_frame
Definition: ReaderBase.h:56
Timeline.h
Header file for Timeline class.
openshot::Clip::ParentTimeline
void ParentTimeline(openshot::TimelineBase *new_timeline) override
Set associated Timeline pointer.
Definition: Clip.cpp:443
openshot::FFmpegReader::~FFmpegReader
virtual ~FFmpegReader()
Destructor.
Definition: FFmpegReader.cpp:137
openshot::ReaderInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
openshot::CacheMemory::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheMemory.cpp:158
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:314
openshot::ReaderInfo::duration
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::ReaderInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
openshot::FFmpegReader::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: FFmpegReader.cpp:3021
openshot::PacketStatus::audio_read
int64_t audio_read
Definition: FFmpegReader.h:51
openshot::ReaderInfo::width
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
openshot::FFmpegReader::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: FFmpegReader.cpp:3045
openshot::PacketStatus::packets_eof
bool packets_eof
Definition: FFmpegReader.h:57
hw_de_av_pix_fmt_global
AVPixelFormat hw_de_av_pix_fmt_global
Definition: FFmpegReader.cpp:72
openshot::PacketStatus::audio_decoded
int64_t audio_decoded
Definition: FFmpegReader.h:52
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::PacketStatus::video_read
int64_t video_read
Definition: FFmpegReader.h:49
hw_de_on
int hw_de_on
Definition: FFmpegReader.cpp:70
openshot::CacheBase::SetMaxBytesFromInfo
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:28
openshot::ReaderBase::MaxDecodeWidth
int MaxDecodeWidth() const
Return the current maximum decoded frame width (0 when unlimited).
Definition: ReaderBase.cpp:257
AV_ALLOCATE_IMAGE
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
Definition: FFmpegUtilities.h:310
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
openshot::Clip::scale_x
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:325
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:321
openshot::ReaderInfo::video_length
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
hw_de_av_device_type_global
AVHWDeviceType hw_de_av_device_type_global
Definition: FFmpegReader.cpp:73
openshot::ReaderInfo::height
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
openshot::PacketStatus::video_eof
bool video_eof
Definition: FFmpegReader.h:55
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
if
if(!codec) codec
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
OPEN_MP_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Definition: OpenMPUtilities.h:23
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:312
openshot::AudioLocation::is_near
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Definition: FFmpegReader.cpp:144
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:261
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::ReaderInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Settings::DE_LIMIT_HEIGHT_MAX
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:86
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:223
openshot::FFmpegReader::enable_seek
bool enable_seek
Definition: FFmpegReader.h:261
openshot::ReaderInfo::file_size
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:153
openshot::FFmpegReader::Open
void Open() override
Open File - which is called by the constructor automatically.
Definition: FFmpegReader.cpp:261
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:354
openshot::SCALE_CROP
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:263
SWRCONTEXT
#define SWRCONTEXT
Definition: FFmpegUtilities.h:264
openshot::PacketStatus::audio_eof
bool audio_eof
Definition: FFmpegReader.h:56
openshot::ReaderInfo::has_single_image
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
openshot::FFmpegReader::final_cache
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:257
openshot::ReaderInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:43
CropHelpers.h
Shared helpers for Crop effect scaling logic.
openshot::ReaderInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
openshot::DurationStrategy::LongestStream
@ LongestStream
Use the longest value from video, audio, or container.
openshot::FFmpegReader
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:103
path
path
Definition: FFmpegWriter.cpp:1474
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:193
openshot::ReaderInfo::audio_stream_index
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::DurationStrategy
DurationStrategy
This enumeration determines which duration source to favor.
Definition: Enums.h:60
openshot::ReaderInfo::audio_timebase
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
openshot::FFmpegReader::Close
void Close() override
Close File.
Definition: FFmpegReader.cpp:748
openshot::SCALE_FIT
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
openshot::PacketStatus::packets_read
int64_t packets_read()
Definition: FFmpegReader.h:60
openshot::ReaderInfo::pixel_format
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::ReaderInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::PacketStatus::packets_decoded
int64_t packets_decoded()
Definition: FFmpegReader.h:65
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:316
openshot::ReaderClosed
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:369
openshot::ReaderInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:315
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:110
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:322
AVCODEC_REGISTER_ALL
#define AVCODEC_REGISTER_ALL
Definition: FFmpegUtilities.h:305
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:262
openshot::Settings::DE_LIMIT_WIDTH_MAX
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:89
openshot::ReaderInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
AV_GET_SAMPLE_FORMAT
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:324
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
openshot::ReaderInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
openshot::PacketStatus::end_of_file
bool end_of_file
Definition: FFmpegReader.h:58
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::Clip::scale
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:180
openshot::ReaderInfo::top_field_first
bool top_field_first
Definition: ReaderBase.h:57
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:163
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:260
openshot::ReaderInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:304
openshot::DurationStrategy::VideoPreferred
@ VideoPreferred
Prefer the video stream's duration, fallback to audio then container.
openshot::CacheMemory::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:100
AV_GET_CODEC_CONTEXT
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:318
openshot::ReaderInfo::video_stream_index
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
openshot::ReaderBase::HasMaxDecodeSize
bool HasMaxDecodeSize() const
Return true when a maximum decoded frame size is active.
Definition: ReaderBase.cpp:265
openshot::FFmpegReader::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: FFmpegReader.cpp:3060
openshot::SCALE_STRETCH
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
openshot::ReaderInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:291
openshot::ReaderInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
openshot::ReaderInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::FFmpegReader::Json
std::string Json() const override
Generate JSON string of this object.
Definition: FFmpegReader.cpp:3014
openshot::FFmpegReader::HardwareDecodeSuccessful
bool HardwareDecodeSuccessful() const override
Return true if hardware decode was requested and successfully produced at least one frame.
Definition: FFmpegReader.cpp:1763
openshot::FFmpegReader::GetIsDurationKnown
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
Definition: FFmpegReader.cpp:1195
openshot::ApplyCropResizeScale
void ApplyCropResizeScale(Clip *clip, int source_width, int source_height, int &max_width, int &max_height)
Scale the requested max_width / max_height based on the Crop resize amount, capped by source size.
Definition: CropHelpers.cpp:40
openshot::PacketStatus::video_decoded
int64_t video_decoded
Definition: FFmpegReader.h:50
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1485
Exceptions.h
Header file for all Exception classes.
openshot::Settings::HW_DE_DEVICE_SET
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:92
FFmpegReader.h
Header file for FFmpegReader class.
openshot::ReaderBase::getFrameMutex
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ReaderBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:243
Generated by   doxygen 1.8.17