OpenShot Library | libopenshot  0.6.0
FFmpegReader.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <thread> // for std::this_thread::sleep_for
17 #include <chrono> // for std::chrono::milliseconds
18 #include <algorithm>
19 #include <cmath>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include "FFmpegUtilities.h"
24 #include "effects/CropHelpers.h"
25 
26 #include "FFmpegReader.h"
27 #include "Exceptions.h"
28 #include "MemoryTrim.h"
29 #include "Timeline.h"
30 #include "ZmqLogger.h"
31 
32 #define ENABLE_VAAPI 0
33 
34 #if USE_HW_ACCEL
35 #define MAX_SUPPORTED_WIDTH 1950
36 #define MAX_SUPPORTED_HEIGHT 1100
37 
38 #if ENABLE_VAAPI
39 #include "libavutil/hwcontext_vaapi.h"
40 
41 typedef struct VAAPIDecodeContext {
42  VAProfile va_profile;
43  VAEntrypoint va_entrypoint;
44  VAConfigID va_config;
45  VAContextID va_context;
46 
47 #if FF_API_STRUCT_VAAPI_CONTEXT
48  // FF_DISABLE_DEPRECATION_WARNINGS
49  int have_old_context;
50  struct vaapi_context *old_context;
51  AVBufferRef *device_ref;
52  // FF_ENABLE_DEPRECATION_WARNINGS
53 #endif
54 
55  AVHWDeviceContext *device;
56  AVVAAPIDeviceContext *hwctx;
57 
58  AVHWFramesContext *frames;
59  AVVAAPIFramesContext *hwfc;
60 
61  enum AVPixelFormat surface_format;
62  int surface_count;
63  } VAAPIDecodeContext;
64 #endif // ENABLE_VAAPI
65 #endif // USE_HW_ACCEL
66 
67 
68 using namespace openshot;
69 
70 int hw_de_on = 0;
71 #if USE_HW_ACCEL
72  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
73  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
74 #endif
75 
76 // Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts.
77 // swscale expects non-YUVJ formats plus explicit color-range metadata.
78 static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) {
79  switch (pix_fmt) {
80  case AV_PIX_FMT_YUVJ420P:
81  is_full_range = true;
82  return AV_PIX_FMT_YUV420P;
83  case AV_PIX_FMT_YUVJ422P:
84  is_full_range = true;
85  return AV_PIX_FMT_YUV422P;
86  case AV_PIX_FMT_YUVJ444P:
87  is_full_range = true;
88  return AV_PIX_FMT_YUV444P;
89  case AV_PIX_FMT_YUVJ440P:
90  is_full_range = true;
91  return AV_PIX_FMT_YUV440P;
92 #ifdef AV_PIX_FMT_YUVJ411P
93  case AV_PIX_FMT_YUVJ411P:
94  is_full_range = true;
95  return AV_PIX_FMT_YUV411P;
96 #endif
97  default:
98  return pix_fmt;
99  }
100 }
101 
102 FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
103  : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {}
104 
105 FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader)
106  : path(path), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
107  pStream(NULL), aStream(NULL), packet(NULL), pFrame(NULL), is_open(false), is_duration_known(false),
108  check_interlace(false), check_fps(false), duration_strategy(duration_strategy), previous_packet_location{-1, 0},
109  is_seeking(false), seeking_pts(0), seeking_frame(0), is_video_seek(true), seek_count(0),
110  seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0),
111  last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0),
112  hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0),
113  NO_PTS_OFFSET(-99999), enable_seek(true) {
114 
115  // Initialize FFMpeg, and register all formats and codecs
116  AV_REGISTER_ALL
117  AVCODEC_REGISTER_ALL
118 
119  // Init timestamp offsets
120  pts_offset_seconds = NO_PTS_OFFSET;
121  video_pts_seconds = NO_PTS_OFFSET;
122  audio_pts_seconds = NO_PTS_OFFSET;
123 
124  // Init cache
125  const int init_working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
126  const int init_final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
127  working_cache.SetMaxBytesFromInfo(init_working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
128  final_cache.SetMaxBytesFromInfo(init_final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
129 
130  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
131  if (inspect_reader) {
132  Open();
133  Close();
134  }
135 }
136 
137 FFmpegReader::~FFmpegReader() {
138  if (is_open)
139  // Auto close reader if not already done
140  Close();
141 }
142 
143 // This struct holds the associated video frame and starting sample # for an audio packet.
144 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
145  // Is frame even close to this one?
146  if (abs(location.frame - frame) >= 2)
147  // This is too far away to be considered
148  return false;
149 
150  // Note that samples_per_frame can vary slightly frame to frame when the
151  // audio sampling rate is not an integer multiple of the video fps.
152  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
153  if (abs(diff) <= amount)
154  // close
155  return true;
156 
157  // not close
158  return false;
159 }
160 
161 #if USE_HW_ACCEL
162 
163 // Get hardware pix format
164 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
165 {
166  const enum AVPixelFormat *p;
167 
168  // Prefer only the format matching the selected hardware decoder
169  int selected = openshot::Settings::Instance()->HARDWARE_DECODER;
170 
171  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
172  switch (*p) {
173 #if defined(__linux__)
174  // Linux pix formats
175  case AV_PIX_FMT_VAAPI:
176  if (selected == 1) {
177  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
178  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
179  return *p;
180  }
181  break;
182  case AV_PIX_FMT_VDPAU:
183  if (selected == 6) {
184  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
185  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
186  return *p;
187  }
188  break;
189 #endif
190 #if defined(_WIN32)
191  // Windows pix formats
192  case AV_PIX_FMT_DXVA2_VLD:
193  if (selected == 3) {
194  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
195  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
196  return *p;
197  }
198  break;
199  case AV_PIX_FMT_D3D11:
200  if (selected == 4) {
201  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
202  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
203  return *p;
204  }
205  break;
206 #endif
207 #if defined(__APPLE__)
208  // Apple pix formats
209  case AV_PIX_FMT_VIDEOTOOLBOX:
210  if (selected == 5) {
211  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
212  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
213  return *p;
214  }
215  break;
216 #endif
217  // Cross-platform pix formats
218  case AV_PIX_FMT_CUDA:
219  if (selected == 2) {
220  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
221  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
222  return *p;
223  }
224  break;
225  case AV_PIX_FMT_QSV:
226  if (selected == 7) {
227  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
228  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
229  return *p;
230  }
231  break;
232  default:
233  // This is only here to silence unused-enum warnings
234  break;
235  }
236  }
237  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
238  return AV_PIX_FMT_NONE;
239 }
240 
241 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
242 {
243  int ret;
244  switch (codecid) {
245  case AV_CODEC_ID_H264:
246  case AV_CODEC_ID_MPEG2VIDEO:
247  case AV_CODEC_ID_VC1:
248  case AV_CODEC_ID_WMV1:
249  case AV_CODEC_ID_WMV2:
250  case AV_CODEC_ID_WMV3:
251  ret = 1;
252  break;
253  default :
254  ret = 0;
255  break;
256  }
257  return ret;
258 }
259 #endif // USE_HW_ACCEL
260 
261 void FFmpegReader::Open() {
262  // Open reader if not already open
263  if (!is_open) {
264  // Prevent async calls to the following code
265  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
266 
267  // Initialize format context
268  pFormatCtx = NULL;
269  {
270  hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0);
271  hw_decode_failed = false;
272  hw_decode_error_count = 0;
273  hw_decode_succeeded = false;
274  ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
275  }
276 
277  // Open video file
278  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
279  throw InvalidFile("FFmpegReader could not open media file.", path);
280 
281  // Retrieve stream information
282  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
283  throw NoStreamsFound("No streams found in file.", path);
284 
285  videoStream = -1;
286  audioStream = -1;
287 
288  // Init end-of-file detection variables
289  packet_status.reset(true);
290 
291  // Loop through each stream, and identify the video and audio stream index
292  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
293  // Is this a video stream?
294  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
295  videoStream = i;
296  packet_status.video_eof = false;
297  packet_status.packets_eof = false;
298  packet_status.end_of_file = false;
299  }
300  // Is this an audio stream?
301  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
302  audioStream = i;
303  packet_status.audio_eof = false;
304  packet_status.packets_eof = false;
305  packet_status.end_of_file = false;
306  }
307  }
308  if (videoStream == -1 && audioStream == -1)
309  throw NoStreamsFound("No video or audio streams found in this file.", path);
310 
311  // Is there a video stream?
312  if (videoStream != -1) {
313  // Set the stream index
314  info.video_stream_index = videoStream;
315 
316  // Set the codec and codec context pointers
317  pStream = pFormatCtx->streams[videoStream];
318 
319  // Find the codec ID from stream
320  const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
321 
322  // Get codec and codec context from stream
323  const AVCodec *pCodec = avcodec_find_decoder(codecId);
324  AVDictionary *opts = NULL;
325  int retry_decode_open = 2;
326  // If hw accel is selected but hardware cannot handle repeat with software decoding
327  do {
328  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
329 #if USE_HW_ACCEL
330  if (hw_de_on && (retry_decode_open==2)) {
331  // Up to here no decision is made if hardware or software decode
332  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
333  }
334 #endif
335  retry_decode_open = 0;
336 
337  // Set number of threads equal to number of processors (not to exceed 16)
338  pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
339 
340  if (pCodec == NULL) {
341  throw InvalidCodec("A valid video codec could not be found for this file.", path);
342  }
343 
344  // Init options
345  av_dict_set(&opts, "strict", "experimental", 0);
346 #if USE_HW_ACCEL
347  if (hw_de_on && hw_de_supported) {
348  // Open Hardware Acceleration
349  int i_decoder_hw = 0;
350  char adapter[256];
351  char *adapter_ptr = NULL;
352  int adapter_num;
353  adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET;
354  ZmqLogger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num);
355 
356  // Set hardware pix format (callback)
357  pCodecCtx->get_format = get_hw_dec_format;
358 
359  if (adapter_num < 3 && adapter_num >=0) {
360 #if defined(__linux__)
361  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
362  adapter_ptr = adapter;
363  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
364  switch (i_decoder_hw) {
365  case 1:
366  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
367  break;
368  case 2:
369  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
370  break;
371  case 6:
372  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
373  break;
374  case 7:
375  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
376  break;
377  default:
378  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
379  break;
380  }
381 
382 #elif defined(_WIN32)
383  adapter_ptr = NULL;
384  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
385  switch (i_decoder_hw) {
386  case 2:
387  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
388  break;
389  case 3:
390  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
391  break;
392  case 4:
393  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
394  break;
395  case 7:
396  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
397  break;
398  default:
399  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
400  break;
401  }
402 #elif defined(__APPLE__)
403  adapter_ptr = NULL;
404  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
405  switch (i_decoder_hw) {
406  case 5:
407  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
408  break;
409  case 7:
410  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
411  break;
412  default:
413  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
414  break;
415  }
416 #endif
417 
418  } else {
419  adapter_ptr = NULL; // Just to be sure
420  }
421 
422  // Check if it is there and writable
423 #if defined(__linux__)
424  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
425 #elif defined(_WIN32)
426  if( adapter_ptr != NULL ) {
427 #elif defined(__APPLE__)
428  if( adapter_ptr != NULL ) {
429 #endif
430  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
431  }
432  else {
433  adapter_ptr = NULL; // use default
434  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
435  }
436 
437  hw_device_ctx = NULL;
438  // Here the first hardware initialisations are made
439  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
440  const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type);
441  std::string hw_msg = "HW decode active: ";
442  hw_msg += (hw_name ? hw_name : "unknown");
443  ZmqLogger::Instance()->Log(hw_msg);
444  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
445  throw InvalidCodec("Hardware device reference create failed.", path);
446  }
447 
448  /*
449  av_buffer_unref(&ist->hw_frames_ctx);
450  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
451  if (!ist->hw_frames_ctx) {
452  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
453  return AVERROR(ENOMEM);
454  }
455 
456  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
457 
458  frames_ctx->format = AV_PIX_FMT_CUDA;
459  frames_ctx->sw_format = avctx->sw_pix_fmt;
460  frames_ctx->width = avctx->width;
461  frames_ctx->height = avctx->height;
462 
463  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
464  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
465 
466 
467  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
468  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
469  if (ret < 0) {
470  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
471  return ret;
472  }
473  */
474  }
475  else {
476  ZmqLogger::Instance()->Log("HW decode active: no (falling back to software)");
477  throw InvalidCodec("Hardware device create failed.", path);
478  }
479  }
480 #endif // USE_HW_ACCEL
481 
482  // Disable per-frame threading for album arts
483  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
484  // but there's only one frame in this case.
485  if (HasAlbumArt())
486  {
487  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
488  }
489 
490  // Open video codec
491  int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
492  if (avcodec_return < 0) {
493  std::stringstream avcodec_error_msg;
494  avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
495  throw InvalidCodec(avcodec_error_msg.str(), path);
496  }
497 
498 #if USE_HW_ACCEL
499  if (hw_de_on && hw_de_supported) {
500  AVHWFramesConstraints *constraints = NULL;
501  void *hwconfig = NULL;
502  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
503 
504 // TODO: needs va_config!
505 #if ENABLE_VAAPI
506  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
507  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
508 #endif // ENABLE_VAAPI
509  if (constraints) {
510  if (pCodecCtx->coded_width < constraints->min_width ||
511  pCodecCtx->coded_height < constraints->min_height ||
512  pCodecCtx->coded_width > constraints->max_width ||
513  pCodecCtx->coded_height > constraints->max_height) {
514  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
515  hw_de_supported = 0;
516  retry_decode_open = 1;
517  AV_FREE_CONTEXT(pCodecCtx);
518  if (hw_device_ctx) {
519  av_buffer_unref(&hw_device_ctx);
520  hw_device_ctx = NULL;
521  }
522  }
523  else {
524  // All is just peachy
525  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
526  retry_decode_open = 0;
527  }
528  av_hwframe_constraints_free(&constraints);
529  if (hwconfig) {
530  av_freep(&hwconfig);
531  }
532  }
533  else {
534  int max_h, max_w;
535  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
536  max_h = openshot::Settings::Instance()->DE_LIMIT_HEIGHT_MAX;
537  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
538  max_w = openshot::Settings::Instance()->DE_LIMIT_WIDTH_MAX;
539  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
540  //cerr << "Constraints could not be found using default limit\n";
541  if (pCodecCtx->coded_width < 0 ||
542  pCodecCtx->coded_height < 0 ||
543  pCodecCtx->coded_width > max_w ||
544  pCodecCtx->coded_height > max_h ) {
545  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
546  hw_de_supported = 0;
547  retry_decode_open = 1;
548  AV_FREE_CONTEXT(pCodecCtx);
549  if (hw_device_ctx) {
550  av_buffer_unref(&hw_device_ctx);
551  hw_device_ctx = NULL;
552  }
553  }
554  else {
555  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
556  retry_decode_open = 0;
557  }
558  }
559  } // if hw_de_on && hw_de_supported
560  else {
561  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
562  }
563 #else
564  retry_decode_open = 0;
565 #endif // USE_HW_ACCEL
566  } while (retry_decode_open); // retry_decode_open
567  // Free options
568  av_dict_free(&opts);
569 
570  // Update the File Info struct with video details (if a video stream is found)
571  UpdateVideoInfo();
572  }
573 
574  // Is there an audio stream?
575  if (audioStream != -1) {
576  // Set the stream index
577  info.audio_stream_index = audioStream;
578 
579  // Get a pointer to the codec context for the audio stream
580  aStream = pFormatCtx->streams[audioStream];
581 
582  // Find the codec ID from stream
583  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
584 
585  // Get codec and codec context from stream
586  const AVCodec *aCodec = avcodec_find_decoder(codecId);
587  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
588 
589  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
590  aCodecCtx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
591 
592  bool audio_opened = false;
593  if (aCodec != NULL) {
594  // Init options
595  AVDictionary *opts = NULL;
596  av_dict_set(&opts, "strict", "experimental", 0);
597 
598  // Open audio codec
599  audio_opened = (avcodec_open2(aCodecCtx, aCodec, &opts) >= 0);
600 
601  // Free options
602  av_dict_free(&opts);
603  }
604 
605  if (audio_opened) {
606  // Update the File Info struct with audio details (if an audio stream is found)
607  UpdateAudioInfo();
608 
609  // Disable malformed audio stream metadata (prevents divide-by-zero / invalid resampling math)
610  const bool invalid_audio_info =
611  (info.channels <= 0) ||
612  (info.sample_rate <= 0) ||
613  (info.audio_timebase.num <= 0) ||
614  (info.audio_timebase.den <= 0) ||
615  (aCodecCtx->sample_fmt == AV_SAMPLE_FMT_NONE);
616  if (invalid_audio_info) {
617  ZmqLogger::Instance()->AppendDebugMethod(
618  "FFmpegReader::Open (Disable invalid audio stream)",
619  "channels", info.channels,
620  "sample_rate", info.sample_rate,
621  "audio_timebase.num", info.audio_timebase.num,
622  "audio_timebase.den", info.audio_timebase.den,
623  "sample_fmt", static_cast<int>(aCodecCtx ? aCodecCtx->sample_fmt : AV_SAMPLE_FMT_NONE));
624  info.has_audio = false;
625  info.audio_stream_index = -1;
626  audioStream = -1;
627  packet_status.audio_eof = true;
628  if (aCodecCtx) {
629  if (avcodec_is_open(aCodecCtx)) {
630  avcodec_flush_buffers(aCodecCtx);
631  }
632  AV_FREE_CONTEXT(aCodecCtx);
633  aCodecCtx = nullptr;
634  }
635  aStream = nullptr;
636  }
637  } else {
638  // Keep decoding video, but disable bad/unsupported audio stream.
639  ZmqLogger::Instance()->AppendDebugMethod(
640  "FFmpegReader::Open (Audio codec unavailable; disabling audio)",
641  "audioStream", audioStream);
642  info.has_audio = false;
643  info.audio_stream_index = -1;
644  audioStream = -1;
645  packet_status.audio_eof = true;
646  if (aCodecCtx) {
647  AV_FREE_CONTEXT(aCodecCtx);
648  aCodecCtx = nullptr;
649  }
650  aStream = nullptr;
651  }
652  }
653 
654  // Guard invalid frame-rate / timebase values from malformed streams.
655  if (info.fps.num <= 0 || info.fps.den <= 0) {
656  ZmqLogger::Instance()->AppendDebugMethod(
657  "FFmpegReader::Open (Invalid FPS detected; applying fallback)",
658  "fps.num", info.fps.num,
659  "fps.den", info.fps.den);
660  info.fps.num = 30;
661  info.fps.den = 1;
662  }
663  if (info.video_timebase.num <= 0 || info.video_timebase.den <= 0) {
664  ZmqLogger::Instance()->AppendDebugMethod(
665  "FFmpegReader::Open (Invalid video_timebase detected; applying fallback)",
666  "video_timebase.num", info.video_timebase.num,
667  "video_timebase.den", info.video_timebase.den);
668  info.video_timebase = info.fps.Reciprocal();
669  }
670 
671  // Add format metadata (if any)
672  AVDictionaryEntry *tag = NULL;
673  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
674  QString str_key = tag->key;
675  QString str_value = tag->value;
676  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
677  }
678 
679  // Process video stream side data (rotation, spherical metadata, etc)
680  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
681  AVStream* st = pFormatCtx->streams[i];
682  if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
683  size_t side_data_size = 0;
684  const uint8_t *displaymatrix = ffmpeg_stream_get_side_data(
685  st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size);
686  if (displaymatrix &&
687  side_data_size >= 9 * sizeof(int32_t) &&
688  !info.metadata.count("rotate")) {
689  double rotation = -av_display_rotation_get(
690  reinterpret_cast<const int32_t *>(displaymatrix));
691  if (std::isnan(rotation))
692  rotation = 0;
693  info.metadata["rotate"] = std::to_string(rotation);
694  }
695 
696  const uint8_t *spherical = ffmpeg_stream_get_side_data(
697  st, AV_PKT_DATA_SPHERICAL, &side_data_size);
698  if (spherical && side_data_size >= sizeof(AVSphericalMapping)) {
699  info.metadata["spherical"] = "1";
700 
701  const AVSphericalMapping *map =
702  reinterpret_cast<const AVSphericalMapping *>(spherical);
703  const char *proj_name = av_spherical_projection_name(map->projection);
704  info.metadata["spherical_projection"] = proj_name ? proj_name : "unknown";
705 
706  auto to_deg = [](int32_t v) {
707  return static_cast<double>(v) / 65536.0;
708  };
709  info.metadata["spherical_yaw"] = std::to_string(to_deg(map->yaw));
710  info.metadata["spherical_pitch"] = std::to_string(to_deg(map->pitch));
711  info.metadata["spherical_roll"] = std::to_string(to_deg(map->roll));
712  }
713  break;
714  }
715  }
716 
717  // Init previous audio location to zero
718  previous_packet_location.frame = -1;
719  previous_packet_location.sample_start = 0;
720 
721  // Adjust cache size based on size of frame and audio
722  const int working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, int(OPEN_MP_NUM_PROCESSORS * info.fps.ToDouble() * 2));
723  const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2);
724  working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
725  final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
726 
727  // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
728  // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
729  UpdatePTSOffset();
730 
731  // Override an invalid framerate
732  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
733  // Calculate FPS, duration, video bit rate, and video length manually
734  // by scanning through all the video stream packets
735  CheckFPS();
736  }
737 
738  // Mark as "open"
739  is_open = true;
740 
741  // Seek back to beginning of file (if not already seeking)
742  if (!is_seeking) {
743  Seek(1);
744  }
745  }
746 }
747 
748 void FFmpegReader::Close() {
749  // Close all objects, if reader is 'open'
750  if (is_open) {
751  // Prevent async calls to the following code
752  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
753 
754  // Mark as "closed"
755  is_open = false;
756 
757  // Keep track of most recent packet
758  AVPacket *recent_packet = packet;
759 
760  // Drain any packets from the decoder
761  packet = NULL;
762  int attempts = 0;
763  int max_attempts = 128;
764  while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
765  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
766  "packets_read", packet_status.packets_read(),
767  "packets_decoded", packet_status.packets_decoded(),
768  "attempts", attempts);
769  if (packet_status.video_decoded < packet_status.video_read) {
770  ProcessVideoPacket(info.video_length);
771  }
772  if (packet_status.audio_decoded < packet_status.audio_read) {
773  ProcessAudioPacket(info.video_length);
774  }
775  attempts++;
776  }
777 
778  // Remove packet
779  if (recent_packet) {
780  RemoveAVPacket(recent_packet);
781  }
782 
783  // Close the video codec
784  if (info.has_video) {
785  if(avcodec_is_open(pCodecCtx)) {
786  avcodec_flush_buffers(pCodecCtx);
787  }
788  AV_FREE_CONTEXT(pCodecCtx);
789 #if USE_HW_ACCEL
790  if (hw_de_on) {
791  if (hw_device_ctx) {
792  av_buffer_unref(&hw_device_ctx);
793  hw_device_ctx = NULL;
794  }
795  }
796 #endif // USE_HW_ACCEL
797  if (img_convert_ctx) {
798  sws_freeContext(img_convert_ctx);
799  img_convert_ctx = nullptr;
800  }
801  if (pFrameRGB_cached) {
802  AV_FREE_FRAME(&pFrameRGB_cached);
803  }
804  }
805 
806  // Close the audio codec
807  if (info.has_audio) {
808  if(avcodec_is_open(aCodecCtx)) {
809  avcodec_flush_buffers(aCodecCtx);
810  }
811  AV_FREE_CONTEXT(aCodecCtx);
812  if (avr_ctx) {
813  SWR_CLOSE(avr_ctx);
814  SWR_FREE(&avr_ctx);
815  avr_ctx = nullptr;
816  }
817  }
818 
819  // Clear final cache
820  final_cache.Clear();
821  working_cache.Clear();
822 
823  // Close the video file
824  avformat_close_input(&pFormatCtx);
825  av_freep(&pFormatCtx);
826 
827  // Do not trim here; trimming is handled on explicit cache clears
828 
829  // Reset some variables
830  last_frame = 0;
831  hold_packet = false;
832  largest_frame_processed = 0;
833  seek_audio_frame_found = 0;
834  seek_video_frame_found = 0;
835  current_video_frame = 0;
836  last_video_frame.reset();
837  last_final_video_frame.reset();
838  }
839 }
840 
841 bool FFmpegReader::HasAlbumArt() {
842  // Check if the video stream we use is an attached picture
843  // This won't return true if the file has a cover image as a secondary stream
844  // like an MKV file with an attached image file
845  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
846  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
847 }
848 
849 double FFmpegReader::PickDurationSeconds() const {
850  auto has_value = [](double value) { return value > 0.0; };
851 
852  switch (duration_strategy) {
853  case DurationStrategy::VideoPreferred:
854  if (has_value(video_stream_duration_seconds))
855  return video_stream_duration_seconds;
856  if (has_value(audio_stream_duration_seconds))
857  return audio_stream_duration_seconds;
858  if (has_value(format_duration_seconds))
859  return format_duration_seconds;
860  break;
861  case DurationStrategy::AudioPreferred:
862  if (has_value(audio_stream_duration_seconds))
863  return audio_stream_duration_seconds;
864  if (has_value(video_stream_duration_seconds))
865  return video_stream_duration_seconds;
866  if (has_value(format_duration_seconds))
867  return format_duration_seconds;
868  break;
869  case DurationStrategy::LongestStream:
870  default:
871  {
872  double longest = 0.0;
873  if (has_value(video_stream_duration_seconds))
874  longest = std::max(longest, video_stream_duration_seconds);
875  if (has_value(audio_stream_duration_seconds))
876  longest = std::max(longest, audio_stream_duration_seconds);
877  if (has_value(format_duration_seconds))
878  longest = std::max(longest, format_duration_seconds);
879  if (has_value(longest))
880  return longest;
881  }
882  break;
883  }
884 
885  if (has_value(format_duration_seconds))
886  return format_duration_seconds;
887  if (has_value(inferred_duration_seconds))
888  return inferred_duration_seconds;
889 
890  return 0.0;
891 }
892 
893 void FFmpegReader::ApplyDurationStrategy() {
894  const double fps_value = info.fps.ToDouble();
895  const double chosen_seconds = PickDurationSeconds();
896 
897  if (chosen_seconds <= 0.0 || fps_value <= 0.0) {
898  info.duration = 0.0f;
899  info.video_length = 0;
900  is_duration_known = false;
901  return;
902  }
903 
904  const int64_t frames = static_cast<int64_t>(std::llround(chosen_seconds * fps_value));
905  if (frames <= 0) {
906  info.duration = 0.0f;
907  info.video_length = 0;
908  is_duration_known = false;
909  return;
910  }
911 
912  info.video_length = frames;
913  info.duration = static_cast<float>(static_cast<double>(frames) / fps_value);
914  is_duration_known = true;
915 }
916 
917 void FFmpegReader::UpdateAudioInfo() {
918  int codec_channels =
919 #if HAVE_CH_LAYOUT
920  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
921 #else
922  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
923 #endif
924 
925  // Set default audio channel layout (if needed)
926 #if HAVE_CH_LAYOUT
927  AVChannelLayout audio_ch_layout = ffmpeg_get_valid_channel_layout(
928  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, codec_channels);
929  if (audio_ch_layout.nb_channels > 0) {
930  av_channel_layout_uninit(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout));
931  av_channel_layout_copy(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout), &audio_ch_layout);
932  codec_channels = audio_ch_layout.nb_channels;
933  }
934 #else
935  if (codec_channels > 0 && AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
936  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
937 #endif
938 
939  if (info.sample_rate > 0) {
940  // Skip init - if info struct already populated
941  return;
942  }
943 
944  auto record_duration = [](double &target, double seconds) {
945  if (seconds > 0.0)
946  target = std::max(target, seconds);
947  };
948 
949  // Set values of FileInfo struct
950  info.has_audio = true;
951  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
952  info.acodec = aCodecCtx->codec->name;
953 #if HAVE_CH_LAYOUT
954  info.channels = audio_ch_layout.nb_channels;
955  info.channel_layout = static_cast<ChannelLayout>(ffmpeg_channel_layout_mask(audio_ch_layout));
956 #else
957  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
958  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
959 #endif
960 
961  // If channel layout is not set, guess based on the number of channels
962  if (info.channel_layout == 0) {
963  if (info.channels == 1) {
964  info.channel_layout = openshot::LAYOUT_MONO;
965  } else if (info.channels == 2) {
966  info.channel_layout = openshot::LAYOUT_STEREO;
967  }
968  }
969 
970  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
971  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
972  if (info.audio_bit_rate <= 0) {
973  // Get bitrate from format
974  info.audio_bit_rate = pFormatCtx->bit_rate;
975  }
976 
977  // Set audio timebase
978  info.audio_timebase.num = aStream->time_base.num;
979  info.audio_timebase.den = aStream->time_base.den;
980 
981  // Get timebase of audio stream (if valid) and greater than the current duration
982  if (aStream->duration > 0) {
983  record_duration(audio_stream_duration_seconds, aStream->duration * info.audio_timebase.ToDouble());
984  }
985  if (pFormatCtx->duration > 0) {
986  // Use the format's duration when stream duration is missing or shorter
987  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
988  }
989 
990  // Calculate duration from filesize and bitrate (if any)
991  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
992  // Estimate from bitrate, total bytes, and framerate
993  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
994  }
995 
996  // Set video timebase (if no video stream was found)
997  if (!info.has_video) {
998  // Set a few important default video settings (so audio can be divided into frames)
999  info.fps.num = 30;
1000  info.fps.den = 1;
1001  info.video_timebase.num = 1;
1002  info.video_timebase.den = 30;
1003  info.width = 720;
1004  info.height = 480;
1005 
1006  // Use timeline to set correct width & height (if any)
1007  Clip *parent = static_cast<Clip *>(ParentClip());
1008  if (parent) {
1009  if (parent->ParentTimeline()) {
1010  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1011  info.width = parent->ParentTimeline()->preview_width;
1012  info.height = parent->ParentTimeline()->preview_height;
1013  }
1014  }
1015  }
1016 
1017  ApplyDurationStrategy();
1018 
1019  // Add audio metadata (if any found)
1020  AVDictionaryEntry *tag = NULL;
1021  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1022  QString str_key = tag->key;
1023  QString str_value = tag->value;
1024  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1025  }
1026 #if HAVE_CH_LAYOUT
1027  av_channel_layout_uninit(&audio_ch_layout);
1028 #endif
1029 }
1030 
1031 void FFmpegReader::UpdateVideoInfo() {
1032  if (info.vcodec.length() > 0) {
1033  // Skip init - if info struct already populated
1034  return;
1035  }
1036 
1037  auto record_duration = [](double &target, double seconds) {
1038  if (seconds > 0.0)
1039  target = std::max(target, seconds);
1040  };
1041 
1042  // Set values of FileInfo struct
1043  info.has_video = true;
1044  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
1045  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
1046  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
1047  info.vcodec = pCodecCtx->codec->name;
1048  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
1049 
1050  // Frame rate from the container and codec
1051  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
1052  if (!check_fps) {
1053  info.fps.num = framerate.num;
1054  info.fps.den = framerate.den;
1055  }
1056 
1057  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
1058 
1059  // TODO: remove excessive debug info in the next releases
1060  // The debug info below is just for comparison and troubleshooting on users side during the transition period
1061  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
1062 
1063  if (pStream->sample_aspect_ratio.num != 0) {
1064  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
1065  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
1066  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
1067  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
1068  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
1069  } else {
1070  info.pixel_ratio.num = 1;
1071  info.pixel_ratio.den = 1;
1072  }
1073  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1074 
1075  // Calculate the DAR (display aspect ratio)
1076  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
1077 
1078  // Reduce size fraction
1079  size.Reduce();
1080 
1081  // Set the ratio based on the reduced fraction
1082  info.display_ratio.num = size.num;
1083  info.display_ratio.den = size.den;
1084 
1085  // Get scan type and order from codec context/params
1086  if (!check_interlace) {
1087  check_interlace = true;
1088  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
1089  switch(field_order) {
1090  case AV_FIELD_PROGRESSIVE:
1091  info.interlaced_frame = false;
1092  break;
1093  case AV_FIELD_TT:
1094  case AV_FIELD_TB:
1095  info.interlaced_frame = true;
1096  info.top_field_first = true;
1097  break;
1098  case AV_FIELD_BT:
1099  case AV_FIELD_BB:
1100  info.interlaced_frame = true;
1101  info.top_field_first = false;
1102  break;
1103  case AV_FIELD_UNKNOWN:
1104  // Check again later?
1105  check_interlace = false;
1106  break;
1107  }
1108  // check_interlace will prevent these checks being repeated,
1109  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
1110  }
1111 
1112  // Set the video timebase
1113  info.video_timebase.num = pStream->time_base.num;
1114  info.video_timebase.den = pStream->time_base.den;
1115 
1116  // Set the duration in seconds, and video length (# of frames)
1117  record_duration(video_stream_duration_seconds, pStream->duration * info.video_timebase.ToDouble());
1118 
1119  // Check for valid duration (if found)
1120  if (pFormatCtx->duration >= 0) {
1121  // Use the format's duration as another candidate
1122  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
1123  }
1124 
1125  // Calculate duration from filesize and bitrate (if any)
1126  if (info.video_bit_rate > 0 && info.file_size > 0) {
1127  // Estimate from bitrate, total bytes, and framerate
1128  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
1129  }
1130 
1131  // Certain "image" formats do not have a valid duration
1132  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1133  pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
1134  // Force an "image" duration
1135  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1136  info.has_single_image = true;
1137  }
1138  // Static GIFs can have no usable duration; fall back to a small default
1139  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1140  pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) {
1141  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1142  info.has_single_image = true;
1143  }
1144 
1145  ApplyDurationStrategy();
1146 
1147  // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior.
1148  // This keeps timing/flags consistent regardless of which reader path was used.
1149  if (!info.has_single_image) {
1150  const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream);
1151  const bool likely_still_codec =
1152  codec_id == AV_CODEC_ID_MJPEG ||
1153  codec_id == AV_CODEC_ID_PNG ||
1154  codec_id == AV_CODEC_ID_BMP ||
1155  codec_id == AV_CODEC_ID_TIFF ||
1156  codec_id == AV_CODEC_ID_WEBP ||
1157  codec_id == AV_CODEC_ID_JPEG2000;
1158  const bool likely_image_demuxer =
1159  pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name &&
1160  strstr(pFormatCtx->iformat->name, "image2");
1161  const bool has_attached_pic = HasAlbumArt();
1162  const bool single_frame_stream =
1163  (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1);
1164  const bool single_frame_clip = info.video_length <= 1;
1165 
1166  const bool is_still_image_video =
1167  has_attached_pic ||
1168  ((single_frame_stream || single_frame_clip) &&
1169  (likely_still_codec || likely_image_demuxer));
1170 
1171  if (is_still_image_video) {
1172  info.has_single_image = true;
1173 
1174  // Only force long duration for standalone images. For audio + attached-art
1175  // files, keep stream-derived duration so the cover image spans the audio.
1176  if (audioStream < 0) {
1177  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1178  }
1179 
1180  ApplyDurationStrategy();
1181  }
1182  }
1183 
1184  // Add video metadata (if any)
1185  AVDictionaryEntry *tag = NULL;
1186  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1187  QString str_key = tag->key;
1188  QString str_value = tag->value;
1189  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1190  }
1191 }
1192 
1193 bool FFmpegReader::GetIsDurationKnown() {
1194  return this->is_duration_known;
1195 }
1196 
1197 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
1198  last_seek_max_frame = -1;
1199  seek_stagnant_count = 0;
1200  // Check for open reader (or throw exception)
1201  if (!is_open)
1202  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
1203 
1204  // Adjust for a requested frame that is too small or too large
1205  if (requested_frame < 1)
1206  requested_frame = 1;
1207  if (requested_frame > info.video_length && is_duration_known)
1208  requested_frame = info.video_length;
1209  if (info.has_video && info.video_length == 0)
1210  // Invalid duration of video file
1211  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
1212 
1213  // Debug output
1214  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
1215 
1216  // Check the cache for this frame
1217  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1218  if (frame) {
1219  // Debug output
1220  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
1221  // Return the cached frame
1222  return frame;
1223  } else {
1224 
1225  // Prevent async calls to the remainder of this code
1226  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
1227 
1228  // Check the cache a 2nd time (due to the potential previous lock)
1229  frame = final_cache.GetFrame(requested_frame);
1230  if (frame) {
1231  // Debug output
1232  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
1233  } else {
1234  // Frame is not in cache
1235  // Reset seek count
1236  seek_count = 0;
1237 
1238  // Are we within X frames of the requested frame?
1239  int64_t diff = requested_frame - last_frame;
1240  if (diff >= 1 && diff <= 20) {
1241  // Continue walking the stream
1242  frame = ReadStream(requested_frame);
1243  } else {
1244  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
1245  if (enable_seek) {
1246  // Only seek if enabled
1247  Seek(requested_frame);
1248 
1249  } else if (!enable_seek && diff < 0) {
1250  // Start over, since we can't seek, and the requested frame is smaller than our position
1251  // Since we are seeking to frame 1, this actually just closes/re-opens the reader
1252  Seek(1);
1253  }
1254 
1255  // Then continue walking the stream
1256  frame = ReadStream(requested_frame);
1257  }
1258  }
1259  return frame;
1260  }
1261 }
1262 
1263 // Read the stream until we find the requested Frame
1264 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
1265  // Allocate video frame
1266  bool check_seek = false;
1267  int packet_error = -1;
1268  int64_t no_progress_count = 0;
1269  int64_t prev_packets_read = packet_status.packets_read();
1270  int64_t prev_packets_decoded = packet_status.packets_decoded();
1271  int64_t prev_video_decoded = packet_status.video_decoded;
1272  double prev_video_pts_seconds = video_pts_seconds;
1273 
1274  // Debug output
1275  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame);
1276 
1277  // Loop through the stream until the correct frame is found
1278  while (true) {
1279  // Check if working frames are 'finished'
1280  if (!is_seeking) {
1281  // Check for final frames
1282  CheckWorkingFrames(requested_frame);
1283  }
1284 
1285  // Check if requested 'final' frame is available (and break out of loop if found)
1286  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1287  if (is_cache_found) {
1288  break;
1289  }
1290 
1291  if (!hold_packet || !packet) {
1292  // Get the next packet
1293  packet_error = GetNextPacket();
1294  if (packet_error < 0 && !packet) {
1295  // No more packets to be found
1296  packet_status.packets_eof = true;
1297  }
1298  }
1299 
1300  // Debug output
1301  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
1302 
1303  // Check the status of a seek (if any)
1304  if (is_seeking) {
1305  check_seek = CheckSeek();
1306  } else {
1307  check_seek = false;
1308  }
1309 
1310  if (check_seek) {
1311  // Packet may become NULL on Close inside Seek if CheckSeek returns false
1312  // Jump to the next iteration of this loop
1313  continue;
1314  }
1315 
1316  // Video packet
1317  if ((info.has_video && packet && packet->stream_index == videoStream) ||
1318  (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1319  (info.has_video && !packet && !packet_status.video_eof)) {
1320  // Process Video Packet
1321  ProcessVideoPacket(requested_frame);
1322  if (ReopenWithoutHardwareDecode(requested_frame)) {
1323  continue;
1324  }
1325  }
1326  // Audio packet
1327  if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1328  (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1329  (info.has_audio && !packet && !packet_status.audio_eof)) {
1330  // Process Audio Packet
1331  ProcessAudioPacket(requested_frame);
1332  }
1333 
1334  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1335  // if the has_video or has_audio properties are manually overridden)
1336  if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1337  (!info.has_audio && packet && packet->stream_index == audioStream)) {
1338  // Keep track of deleted packet counts
1339  if (packet->stream_index == videoStream) {
1340  packet_status.video_decoded++;
1341  } else if (packet->stream_index == audioStream) {
1342  packet_status.audio_decoded++;
1343  }
1344 
1345  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1346  // if the has_video or has_audio properties are manually overridden)
1347  RemoveAVPacket(packet);
1348  packet = NULL;
1349  }
1350 
1351  // Determine end-of-stream (waiting until final decoder threads finish)
1352  // Force end-of-stream in some situations
1353  packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1354  if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1355  // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1356  // If we have no more packets, and all known packets have been decoded
1357  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1358  if (!packet_status.video_eof) {
1359  packet_status.video_eof = true;
1360  }
1361  if (!packet_status.audio_eof) {
1362  packet_status.audio_eof = true;
1363  }
1364  packet_status.end_of_file = true;
1365  break;
1366  }
1367 
1368  // Detect decoder stalls with no progress at EOF and force completion so
1369  // missing frames can be finalized from prior image data.
1370  const bool has_progress =
1371  (packet_status.packets_read() != prev_packets_read) ||
1372  (packet_status.packets_decoded() != prev_packets_decoded) ||
1373  (packet_status.video_decoded != prev_video_decoded) ||
1374  (video_pts_seconds != prev_video_pts_seconds);
1375 
1376  if (has_progress) {
1377  no_progress_count = 0;
1378  } else {
1379  no_progress_count++;
1380  if (no_progress_count >= 2000
1381  && packet_status.packets_eof
1382  && !packet
1383  && !hold_packet) {
1384  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF after stall)",
1385  "requested_frame", requested_frame,
1386  "no_progress_count", no_progress_count,
1387  "packets_read", packet_status.packets_read(),
1388  "packets_decoded", packet_status.packets_decoded(),
1389  "video_decoded", packet_status.video_decoded,
1390  "audio_decoded", packet_status.audio_decoded);
1391  packet_status.video_eof = true;
1392  packet_status.audio_eof = true;
1393  packet_status.end_of_file = true;
1394  break;
1395  }
1396  }
1397  prev_packets_read = packet_status.packets_read();
1398  prev_packets_decoded = packet_status.packets_decoded();
1399  prev_video_decoded = packet_status.video_decoded;
1400  prev_video_pts_seconds = video_pts_seconds;
1401  } // end while
1402 
1403  // Debug output
1404  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1405  "packets_read", packet_status.packets_read(),
1406  "packets_decoded", packet_status.packets_decoded(),
1407  "end_of_file", packet_status.end_of_file,
1408  "largest_frame_processed", largest_frame_processed,
1409  "Working Cache Count", working_cache.Count());
1410 
1411  // Have we reached end-of-stream (or the final frame)?
1412  if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1413  // Force end-of-stream
1414  packet_status.end_of_file = true;
1415  }
1416  if (packet_status.end_of_file) {
1417  // Mark any other working frames as 'finished'
1418  CheckWorkingFrames(requested_frame);
1419  }
1420 
1421  // Return requested frame (if found)
1422  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1423  if (frame)
1424  // Return prepared frame
1425  return frame;
1426  else {
1427 
1428  // Check if largest frame is still cached
1429  frame = final_cache.GetFrame(largest_frame_processed);
1430  int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1431  info.sample_rate, info.channels);
1432  if (frame) {
1433  // Copy and return the largest processed frame (assuming it was the last in the video file)
1434  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1435  if (frame->has_image_data) {
1436  f->AddImage(std::make_shared<QImage>(frame->GetImage()->copy()));
1437  }
1438 
1439  // Use solid color (if no image data found)
1440  if (!frame->has_image_data) {
1441  // Use solid black frame if no image data available
1442  f->AddColor(info.width, info.height, "#000");
1443  }
1444  // Silence audio data (if any), since we are repeating the last frame
1445  f->AddAudioSilence(samples_in_frame);
1446 
1447  return f;
1448  } else {
1449  // The largest processed frame is no longer in cache. Prefer the most recent
1450  // finalized image first, then decoded image, to avoid black flashes.
1451  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1452  if (last_final_video_frame && last_final_video_frame->has_image_data
1453  && last_final_video_frame->number <= requested_frame) {
1454  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
1455  } else if (last_video_frame && last_video_frame->has_image_data
1456  && last_video_frame->number <= requested_frame) {
1457  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
1458  } else {
1459  f->AddColor(info.width, info.height, "#000");
1460  }
1461  f->AddAudioSilence(samples_in_frame);
1462  return f;
1463  }
1464  }
1465 
1466 }
1467 
1468 // Get the next packet (if any)
1469 int FFmpegReader::GetNextPacket() {
1470  int found_packet = 0;
1471  AVPacket *next_packet;
1472  next_packet = new AVPacket();
1473  found_packet = av_read_frame(pFormatCtx, next_packet);
1474 
1475  if (packet) {
1476  // Remove previous packet before getting next one
1477  RemoveAVPacket(packet);
1478  packet = NULL;
1479  }
1480  if (found_packet >= 0) {
1481  // Update current packet pointer
1482  packet = next_packet;
1483 
1484  // Keep track of packet stats
1485  if (packet->stream_index == videoStream) {
1486  packet_status.video_read++;
1487  } else if (packet->stream_index == audioStream) {
1488  packet_status.audio_read++;
1489  }
1490  } else {
1491  // No more packets found
1492  delete next_packet;
1493  packet = NULL;
1494  }
1495  // Return if packet was found (or error number)
1496  return found_packet;
1497 }
1498 
1499 // Get an AVFrame (if any)
1500 bool FFmpegReader::GetAVFrame() {
1501  int frameFinished = 0;
1502  auto note_hw_decode_failure = [&](int err, const char* stage) {
1503 #if USE_HW_ACCEL
1504  if (!hw_de_on || !hw_de_supported || force_sw_decode) {
1505  return;
1506  }
1507  if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) {
1508  hw_decode_error_count++;
1509  ZmqLogger::Instance()->AppendDebugMethod(
1510  std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")",
1511  "error_count", hw_decode_error_count,
1512  "error", err);
1513  if (hw_decode_error_count >= 3) {
1514  hw_decode_failed = true;
1515  }
1516  }
1517 #else
1518  (void) err;
1519  (void) stage;
1520 #endif
1521  };
1522 
1523  // Decode video frame
1524  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1525 
1526 #if IS_FFMPEG_3_2
1527  int send_packet_err = 0;
1528  int64_t send_packet_pts = 0;
1529  if ((packet && packet->stream_index == videoStream) || !packet) {
1530  send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1531 
1532  if (packet && send_packet_err >= 0) {
1533  send_packet_pts = GetPacketPTS();
1534  hold_packet = false;
1535  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1536  }
1537  }
1538 
1539  #if USE_HW_ACCEL
1540  // Get the format from the variables set in get_hw_dec_format
1541  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1542  hw_de_av_device_type = hw_de_av_device_type_global;
1543  #endif // USE_HW_ACCEL
1544  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1545  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1546  note_hw_decode_failure(send_packet_err, "send_packet");
1547  if (send_packet_err == AVERROR(EAGAIN)) {
1548  hold_packet = true;
1549  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1550  }
1551  if (send_packet_err == AVERROR(EINVAL)) {
1552  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1553  }
1554  if (send_packet_err == AVERROR(ENOMEM)) {
1555  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1556  }
1557  }
1558 
1559  // Always try and receive a packet, if not EOF.
1560  // Even if the above avcodec_send_packet failed to send,
1561  // we might still need to receive a packet.
1562  int receive_frame_err = 0;
1563  AVFrame *decoded_frame = next_frame;
1564  AVFrame *next_frame2;
1565 #if USE_HW_ACCEL
1566  if (hw_de_on && hw_de_supported) {
1567  next_frame2 = AV_ALLOCATE_FRAME();
1568  }
1569  else
1570 #endif // USE_HW_ACCEL
1571  {
1572  next_frame2 = next_frame;
1573  }
1574  pFrame = AV_ALLOCATE_FRAME();
1575  while (receive_frame_err >= 0) {
1576  receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1577 
1578  if (receive_frame_err != 0) {
1579  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1580  note_hw_decode_failure(receive_frame_err, "receive_frame");
1581 
1582  if (receive_frame_err == AVERROR_EOF) {
1583  ZmqLogger::Instance()->AppendDebugMethod(
1584  "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1585  avcodec_flush_buffers(pCodecCtx);
1586  packet_status.video_eof = true;
1587  }
1588  if (receive_frame_err == AVERROR(EINVAL)) {
1589  ZmqLogger::Instance()->AppendDebugMethod(
1590  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1591  avcodec_flush_buffers(pCodecCtx);
1592  }
1593  if (receive_frame_err == AVERROR(EAGAIN)) {
1594  ZmqLogger::Instance()->AppendDebugMethod(
1595  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1596  }
1597  if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1598  ZmqLogger::Instance()->AppendDebugMethod(
1599  "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1600  }
1601 
1602  // Break out of decoding loop
1603  // Nothing ready for decoding yet
1604  break;
1605  }
1606 
1607 #if USE_HW_ACCEL
1608  if (hw_de_on && hw_de_supported) {
1609  int err;
1610  if (next_frame2->format == hw_de_av_pix_fmt) {
1611  if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) {
1612  ZmqLogger::Instance()->AppendDebugMethod(
1613  "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)",
1614  "hw_de_on", hw_de_on,
1615  "error", err);
1616  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer");
1617  break;
1618  }
1619  if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) {
1620  ZmqLogger::Instance()->AppendDebugMethod(
1621  "FFmpegReader::GetAVFrame (Failed to copy props to output frame)",
1622  "hw_de_on", hw_de_on,
1623  "error", err);
1624  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props");
1625  break;
1626  }
1627  if (next_frame->format == AV_PIX_FMT_NONE) {
1628  next_frame->format = pCodecCtx->sw_pix_fmt;
1629  }
1630  if (next_frame->width <= 0) {
1631  next_frame->width = next_frame2->width;
1632  }
1633  if (next_frame->height <= 0) {
1634  next_frame->height = next_frame2->height;
1635  }
1636  decoded_frame = next_frame;
1637  } else {
1638  // Some hardware decoders can still return software-readable frames.
1639  decoded_frame = next_frame2;
1640  }
1641  }
1642  else
1643 #endif // USE_HW_ACCEL
1644  { // No hardware acceleration used -> no copy from GPU memory needed
1645  decoded_frame = next_frame2;
1646  }
1647 
1648  if (!decoded_frame->data[0]) {
1649  ZmqLogger::Instance()->AppendDebugMethod(
1650  "FFmpegReader::GetAVFrame (Decoded frame missing image data)",
1651  "format", decoded_frame->format,
1652  "width", decoded_frame->width,
1653  "height", decoded_frame->height);
1654  note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty");
1655  break;
1656  }
1657 
1658  // TODO also handle possible further frames
1659  // Use only the first frame like avcodec_decode_video2
1660  frameFinished = 1;
1661  hw_decode_error_count = 0;
1662 #if USE_HW_ACCEL
1663  if (hw_de_on && hw_de_supported && !force_sw_decode) {
1664  hw_decode_succeeded = true;
1665  }
1666 #endif
1667  packet_status.video_decoded++;
1668 
1669  // Allocate image (align 32 for simd)
1670  AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
1671  if (decoded_pix_fmt == AV_PIX_FMT_NONE)
1672  decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
1673  if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) {
1674  throw OutOfMemory("Failed to allocate image buffer", path);
1675  }
1676  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize,
1677  decoded_pix_fmt, info.width, info.height);
1678  pFrame->format = decoded_pix_fmt;
1679  pFrame->width = info.width;
1680  pFrame->height = info.height;
1681  pFrame->color_range = decoded_frame->color_range;
1682  pFrame->colorspace = decoded_frame->colorspace;
1683  pFrame->color_primaries = decoded_frame->color_primaries;
1684  pFrame->color_trc = decoded_frame->color_trc;
1685  pFrame->chroma_location = decoded_frame->chroma_location;
1686 
1687  // Get display PTS from video frame, often different than packet->pts.
1688  // Sending packets to the decoder (i.e. packet->pts) is async,
1689  // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1690  // sending and retrieving are separated by multiple calls to this method.
1691  if (decoded_frame->pts != AV_NOPTS_VALUE) {
1692  // This is the current decoded frame (and should be the pts used) for
1693  // processing this data
1694  video_pts = decoded_frame->pts;
1695  } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) {
1696  // Some videos only set this timestamp (fallback)
1697  video_pts = decoded_frame->pkt_dts;
1698  }
1699 
1700  ZmqLogger::Instance()->AppendDebugMethod(
1701  "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1702 
1703  // break out of loop after each successful image returned
1704  break;
1705  }
1706 #if USE_HW_ACCEL
1707  if (hw_de_on && hw_de_supported && next_frame2 != next_frame) {
1708  AV_FREE_FRAME(&next_frame2);
1709  }
1710  #endif // USE_HW_ACCEL
1711 #else
1712  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1713 
1714  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1715  pFrame = AV_ALLOCATE_FRAME();
1716 
1717  // is frame finished
1718  if (frameFinished) {
1719  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1720  // must make a copy of the image data before this method is called again.
1721  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1722  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1723  info.height);
1724  }
1725 #endif // IS_FFMPEG_3_2
1726 
1727  // deallocate the frame
1728  AV_FREE_FRAME(&next_frame);
1729 
1730  // Did we get a video frame?
1731  return frameFinished;
1732 }
1733 
1734 bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) {
1735 #if USE_HW_ACCEL
1736  if (!hw_decode_failed || force_sw_decode) {
1737  return false;
1738  }
1739 
1740  ZmqLogger::Instance()->AppendDebugMethod(
1741  "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)",
1742  "requested_frame", requested_frame,
1743  "video_packets_read", packet_status.video_read,
1744  "video_packets_decoded", packet_status.video_decoded,
1745  "hw_decode_error_count", hw_decode_error_count);
1746 
1747  force_sw_decode = true;
1748  hw_decode_failed = false;
1749  hw_decode_error_count = 0;
1750 
1751  Close();
1752  Open();
1753  Seek(requested_frame);
1754  return true;
1755 #else
1756  (void) requested_frame;
1757  return false;
1758 #endif
1759 }
1760 
1761 bool FFmpegReader::HardwareDecodeSuccessful() const {
1762 #if USE_HW_ACCEL
1763  return hw_decode_succeeded;
1764 #else
1765  return false;
1766 #endif
1767 }
1768 
1769 // Check the current seek position and determine if we need to seek again
1770 bool FFmpegReader::CheckSeek() {
1771  // Are we seeking for a specific frame?
1772  if (is_seeking) {
1773  const int64_t kSeekRetryMax = 5;
1774  const int kSeekStagnantMax = 2;
1775 
1776  // Determine if both an audio and video packet have been decoded since the seek happened.
1777  // If not, allow the ReadStream method to keep looping
1778  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1779  return false;
1780 
1781  // Check for both streams
1782  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1783  return false;
1784 
1785  // Determine max seeked frame
1786  int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1787  // Track stagnant seek results (no progress between retries)
1788  if (max_seeked_frame == last_seek_max_frame) {
1789  seek_stagnant_count++;
1790  } else {
1791  last_seek_max_frame = max_seeked_frame;
1792  seek_stagnant_count = 0;
1793  }
1794 
1795  // determine if we are "before" the requested frame
1796  if (max_seeked_frame >= seeking_frame) {
1797  // SEEKED TOO FAR
1798  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1799  "is_video_seek", is_video_seek,
1800  "max_seeked_frame", max_seeked_frame,
1801  "seeking_frame", seeking_frame,
1802  "seeking_pts", seeking_pts,
1803  "seek_video_frame_found", seek_video_frame_found,
1804  "seek_audio_frame_found", seek_audio_frame_found);
1805 
1806  // Seek again... to the nearest Keyframe
1807  if (seek_count < kSeekRetryMax) {
1808  Seek(seeking_frame - (10 * seek_count * seek_count));
1809  } else if (seek_stagnant_count >= kSeekStagnantMax) {
1810  // Stagnant seek: force a much earlier target and keep seeking.
1811  Seek(seeking_frame - (10 * kSeekRetryMax * kSeekRetryMax));
1812  } else {
1813  // Retry budget exhausted: keep seeking from a conservative offset.
1814  Seek(seeking_frame - (10 * seek_count * seek_count));
1815  }
1816  } else {
1817  // SEEK WORKED
1818  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1819  "is_video_seek", is_video_seek,
1820  "packet->pts", GetPacketPTS(),
1821  "seeking_pts", seeking_pts,
1822  "seeking_frame", seeking_frame,
1823  "seek_video_frame_found", seek_video_frame_found,
1824  "seek_audio_frame_found", seek_audio_frame_found);
1825 
1826  // Seek worked, and we are "before" the requested frame
1827  is_seeking = false;
1828  seeking_frame = 0;
1829  seeking_pts = -1;
1830  }
1831  }
1832 
1833  // return the pts to seek to (if any)
1834  return is_seeking;
1835 }
1836 
1837 // Process a video packet
1838 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1839  // Get the AVFrame from the current packet
1840  // This sets the video_pts to the correct timestamp
1841  int frame_finished = GetAVFrame();
1842 
1843  // Check if the AVFrame is finished and set it
1844  if (!frame_finished) {
1845  // No AVFrame decoded yet, bail out
1846  if (pFrame) {
1847  RemoveAVFrame(pFrame);
1848  }
1849  return;
1850  }
1851 
1852  // Calculate current frame #
1853  int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1854 
1855  // Track 1st video packet after a successful seek
1856  if (!seek_video_frame_found && is_seeking)
1857  seek_video_frame_found = current_frame;
1858 
1859  // Create or get the existing frame object. Requested frame needs to be created
1860  // in working_cache at least once. Seek can clear the working_cache, so we must
1861  // add the requested frame back to the working_cache here. If it already exists,
1862  // it will be moved to the top of the working_cache.
1863  working_cache.Add(CreateFrame(requested_frame));
1864 
1865  // Debug output
1866  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1867 
1868  // Init some things local (for OpenMP)
1869  AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE)
1870  ? static_cast<AVPixelFormat>(pFrame->format)
1871  : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1872  bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG);
1873  AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range);
1874  int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width;
1875  int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height;
1876  int height = src_height;
1877  int width = src_width;
1878  // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it)
1879  AVFrame *pFrameRGB = pFrameRGB_cached;
1880  if (!pFrameRGB) {
1881  pFrameRGB = AV_ALLOCATE_FRAME();
1882  if (pFrameRGB == nullptr)
1883  throw OutOfMemory("Failed to allocate frame buffer", path);
1884  pFrameRGB_cached = pFrameRGB;
1885  }
1886  AV_RESET_FRAME(pFrameRGB);
1887  uint8_t *buffer = nullptr;
1888 
1889  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1890  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1891  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1892  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1893  // the future.
1894  int max_width = info.width;
1895  int max_height = info.height;
1896 
1897  Clip *parent = static_cast<Clip *>(ParentClip());
1898  if (parent) {
1899  if (parent->ParentTimeline()) {
1900  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1901  max_width = parent->ParentTimeline()->preview_width;
1902  max_height = parent->ParentTimeline()->preview_height;
1903  }
1904  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1905  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1906  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1907  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1908  max_width = std::max(float(max_width), max_width * max_scale_x);
1909  max_height = std::max(float(max_height), max_height * max_scale_y);
1910 
1911  } else if (parent->scale == SCALE_CROP) {
1912  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1913  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1914  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1915  QSize width_size(max_width * max_scale_x,
1916  round(max_width / (float(info.width) / float(info.height))));
1917  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1918  max_height * max_scale_y);
1919  // respect aspect ratio
1920  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1921  max_width = std::max(max_width, width_size.width());
1922  max_height = std::max(max_height, width_size.height());
1923  } else {
1924  max_width = std::max(max_width, height_size.width());
1925  max_height = std::max(max_height, height_size.height());
1926  }
1927 
1928  } else {
1929  // Scale video to equivalent unscaled size
1930  // Since the preview window can change sizes, we want to always
1931  // scale against the ratio of original video size to timeline size
1932  float preview_ratio = 1.0;
1933  if (parent->ParentTimeline()) {
1934  Timeline *t = (Timeline *) parent->ParentTimeline();
1935  preview_ratio = t->preview_width / float(t->info.width);
1936  }
1937  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1938  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1939  max_width = info.width * max_scale_x * preview_ratio;
1940  max_height = info.height * max_scale_y * preview_ratio;
1941  }
1942 
1943  // If a crop effect is resizing the image, request enough pixels to preserve detail
1944  ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height);
1945  }
1946 
1947  // Determine if image needs to be scaled (for performance reasons)
1948  int original_height = src_height;
1949  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1950  // Override width and height (but maintain aspect ratio)
1951  float ratio = float(width) / float(height);
1952  int possible_width = round(max_height * ratio);
1953  int possible_height = round(max_width / ratio);
1954 
1955  if (possible_width <= max_width) {
1956  // use calculated width, and max_height
1957  width = possible_width;
1958  height = max_height;
1959  } else {
1960  // use max_width, and calculated height
1961  width = max_width;
1962  height = possible_height;
1963  }
1964  }
1965 
1966  // Determine required buffer size and allocate buffer
1967  const int bytes_per_pixel = 4;
1968  int raw_buffer_size = (width * height * bytes_per_pixel) + 128;
1969 
1970  // Aligned memory allocation (for speed)
1971  constexpr size_t ALIGNMENT = 32; // AVX2
1972  int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
1973  buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT);
1974 
1975  // Copy picture data from one AVFrame (or AVPicture) to another one.
1976  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1977 
1978  int scale_mode = SWS_FAST_BILINEAR;
1979  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1980  scale_mode = SWS_BICUBIC;
1981  }
1982  img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1983  if (!img_convert_ctx)
1984  throw OutOfMemory("Failed to initialize sws context", path);
1985  const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
1986  const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
1987  const int dst_full_range = 1; // RGB outputs are full-range
1988  sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0,
1989  dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16);
1990 
1991  if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) {
1992 #if USE_HW_ACCEL
1993  if (hw_de_on && hw_de_supported && !force_sw_decode) {
1994  hw_decode_failed = true;
1995  ZmqLogger::Instance()->AppendDebugMethod(
1996  "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)",
1997  "requested_frame", requested_frame,
1998  "current_frame", current_frame,
1999  "src_pix_fmt", src_pix_fmt,
2000  "src_width", src_width,
2001  "src_height", src_height);
2002  }
2003 #endif
2004  if (pFrame) {
2005  RemoveAVFrame(pFrame);
2006  pFrame = NULL;
2007  }
2008  return;
2009  }
2010 
2011  // Resize / Convert to RGB
2012  const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
2013  original_height, pFrameRGB->data, pFrameRGB->linesize);
2014  if (scaled_lines <= 0) {
2015 #if USE_HW_ACCEL
2016  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2017  hw_decode_failed = true;
2018  ZmqLogger::Instance()->AppendDebugMethod(
2019  "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)",
2020  "requested_frame", requested_frame,
2021  "current_frame", current_frame,
2022  "scaled_lines", scaled_lines,
2023  "src_pix_fmt", src_pix_fmt,
2024  "src_width", src_width,
2025  "src_height", src_height);
2026  }
2027 #endif
2028  free(buffer);
2029  AV_RESET_FRAME(pFrameRGB);
2030  RemoveAVFrame(pFrame);
2031  pFrame = NULL;
2032  return;
2033  }
2034 
2035  // Create or get the existing frame object
2036  std::shared_ptr<Frame> f = CreateFrame(current_frame);
2037 
2038  // Add Image data to frame
2039  if (!ffmpeg_has_alpha(src_pix_fmt)) {
2040  // Add image with no alpha channel, Speed optimization
2041  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
2042  } else {
2043  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
2044  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
2045  }
2046 
2047  // Update working cache
2048  working_cache.Add(f);
2049 
2050  // Keep track of last last_video_frame
2051  last_video_frame = f;
2052 
2053  // Free the RGB image
2054  AV_RESET_FRAME(pFrameRGB);
2055 
2056  // Remove frame and packet
2057  RemoveAVFrame(pFrame);
2058 
2059  // Get video PTS in seconds
2060  video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2061 
2062  // Debug output
2063  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
2064 }
2065 
2066 // Process an audio packet
2067 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
2068  AudioLocation location;
2069  // Calculate location of current audio packet
2070  if (packet && packet->pts != AV_NOPTS_VALUE) {
2071  // Determine related video frame and starting sample # from audio PTS
2072  location = GetAudioPTSLocation(packet->pts);
2073 
2074  // Track 1st audio packet after a successful seek
2075  if (!seek_audio_frame_found && is_seeking)
2076  seek_audio_frame_found = location.frame;
2077  }
2078 
2079  // Create or get the existing frame object. Requested frame needs to be created
2080  // in working_cache at least once. Seek can clear the working_cache, so we must
2081  // add the requested frame back to the working_cache here. If it already exists,
2082  // it will be moved to the top of the working_cache.
2083  working_cache.Add(CreateFrame(requested_frame));
2084 
2085  // Debug output
2086  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
2087  "requested_frame", requested_frame,
2088  "target_frame", location.frame,
2089  "starting_sample", location.sample_start);
2090 
2091  // Init an AVFrame to hold the decoded audio samples
2092  int frame_finished = 0;
2093  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
2094  AV_RESET_FRAME(audio_frame);
2095 
2096  int packet_samples = 0;
2097  int data_size = 0;
2098 
2099 #if IS_FFMPEG_3_2
2100  int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
2101  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
2102  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
2103  }
2104  else {
2105  int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
2106  if (receive_frame_err >= 0) {
2107  frame_finished = 1;
2108  }
2109  if (receive_frame_err == AVERROR_EOF) {
2110  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
2111  packet_status.audio_eof = true;
2112  }
2113  if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
2114  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
2115  avcodec_flush_buffers(aCodecCtx);
2116  }
2117  if (receive_frame_err != 0) {
2118  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
2119  }
2120  }
2121 #else
2122  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
2123 #endif
2124 
2125  if (frame_finished) {
2126  packet_status.audio_decoded++;
2127 
2128  // This can be different than the current packet, so we need to look
2129  // at the current AVFrame from the audio decoder. This timestamp should
2130  // be used for the remainder of this function
2131  audio_pts = audio_frame->pts;
2132 
2133  // Determine related video frame and starting sample # from audio PTS
2134  location = GetAudioPTSLocation(audio_pts);
2135 
2136  // determine how many samples were decoded
2137  int plane_size = -1;
2138 #if HAVE_CH_LAYOUT
2139  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
2140 #else
2141  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
2142 #endif
2143  data_size = av_samples_get_buffer_size(&plane_size, nb_channels,
2144  audio_frame->nb_samples, (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
2145 
2146  // Calculate total number of samples
2147  packet_samples = audio_frame->nb_samples * nb_channels;
2148  } else {
2149  if (audio_frame) {
2150  // Free audio frame
2151  AV_FREE_FRAME(&audio_frame);
2152  }
2153  }
2154 
2155  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
2156  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
2157 
2158  // Bail if no samples found
2159  if (pts_remaining_samples == 0) {
2160  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
2161  "packet_samples", packet_samples,
2162  "info.channels", info.channels,
2163  "pts_remaining_samples", pts_remaining_samples);
2164  return;
2165  }
2166 
2167  while (pts_remaining_samples) {
2168  // Get Samples per frame (for this frame number)
2169  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
2170 
2171  // Calculate # of samples to add to this frame
2172  int samples = samples_per_frame - previous_packet_location.sample_start;
2173  if (samples > pts_remaining_samples)
2174  samples = pts_remaining_samples;
2175 
2176  // Decrement remaining samples
2177  pts_remaining_samples -= samples;
2178 
2179  if (pts_remaining_samples > 0) {
2180  // next frame
2181  previous_packet_location.frame++;
2182  previous_packet_location.sample_start = 0;
2183  } else {
2184  // Increment sample start
2185  previous_packet_location.sample_start += samples;
2186  }
2187  }
2188 
2189  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
2190  "packet_samples", packet_samples,
2191  "info.channels", info.channels,
2192  "info.sample_rate", info.sample_rate,
2193  "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx));
2194 
2195  // Create output frame
2196  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
2197  AV_RESET_FRAME(audio_converted);
2198  audio_converted->nb_samples = audio_frame->nb_samples;
2199  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_FLTP, 0);
2200 
2201  SWRCONTEXT *avr = avr_ctx;
2202  // setup resample context if needed
2203  if (!avr) {
2204  avr = SWR_ALLOC();
2205 #if HAVE_CH_LAYOUT
2206  AVChannelLayout input_layout = ffmpeg_get_valid_channel_layout(
2207  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, info.channels);
2208  AVChannelLayout output_layout = ffmpeg_get_valid_channel_layout(
2209  input_layout, info.channels);
2210  int in_layout_err = av_opt_set_chlayout(avr, "in_chlayout", &input_layout, 0);
2211  int out_layout_err = av_opt_set_chlayout(avr, "out_chlayout", &output_layout, 0);
2212 #else
2213  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2214  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2215  av_opt_set_int(avr, "in_channels", info.channels, 0);
2216  av_opt_set_int(avr, "out_channels", info.channels, 0);
2217 #endif
2218  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
2219  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
2220  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
2221  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
2222  int swr_init_err = SWR_INIT(avr);
2223 #if HAVE_CH_LAYOUT
2224  av_channel_layout_uninit(&input_layout);
2225  av_channel_layout_uninit(&output_layout);
2226  if (in_layout_err < 0 || out_layout_err < 0 || swr_init_err < 0) {
2227  SWR_FREE(&avr);
2228  throw InvalidChannels("Could not initialize FFmpeg audio channel layout or resampler.", path);
2229  }
2230 #else
2231  if (swr_init_err < 0) {
2232  SWR_FREE(&avr);
2233  throw InvalidChannels("Could not initialize FFmpeg audio resampler.", path);
2234  }
2235 #endif
2236  avr_ctx = avr;
2237  }
2238 
2239  // Convert audio samples
2240  int nb_samples = SWR_CONVERT(avr, // audio resample context
2241  audio_converted->data, // output data pointers
2242  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
2243  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
2244  audio_frame->data, // input data pointers
2245  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
2246  audio_frame->nb_samples); // number of input samples to convert
2247 
2248 
2249  int64_t starting_frame_number = -1;
2250  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
2251  // Array of floats (to hold samples for each channel)
2252  starting_frame_number = location.frame;
2253  int channel_buffer_size = nb_samples;
2254  auto *channel_buffer = (float *) (audio_converted->data[channel_filter]);
2255 
2256  // Loop through samples, and add them to the correct frames
2257  int start = location.sample_start;
2258  int remaining_samples = channel_buffer_size;
2259  while (remaining_samples > 0) {
2260  // Get Samples per frame (for this frame number)
2261  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
2262 
2263  // Calculate # of samples to add to this frame
2264  int samples = std::fmin(samples_per_frame - start, remaining_samples);
2265 
2266  // Create or get the existing frame object
2267  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
2268 
2269  // Add samples for current channel to the frame.
2270  f->AddAudio(true, channel_filter, start, channel_buffer, samples, 1.0f);
2271 
2272  // Debug output
2273  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
2274  "frame", starting_frame_number,
2275  "start", start,
2276  "samples", samples,
2277  "channel", channel_filter,
2278  "samples_per_frame", samples_per_frame);
2279 
2280  // Add or update cache
2281  working_cache.Add(f);
2282 
2283  // Decrement remaining samples
2284  remaining_samples -= samples;
2285 
2286  // Increment buffer (to next set of samples)
2287  if (remaining_samples > 0)
2288  channel_buffer += samples;
2289 
2290  // Increment frame number
2291  starting_frame_number++;
2292 
2293  // Reset starting sample #
2294  start = 0;
2295  }
2296  }
2297 
2298  // Free AVFrames
2299  av_free(audio_converted->data[0]);
2300  AV_FREE_FRAME(&audio_converted);
2301  AV_FREE_FRAME(&audio_frame);
2302 
2303  // Get audio PTS in seconds
2304  audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2305 
2306  // Debug output
2307  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
2308  "requested_frame", requested_frame,
2309  "starting_frame", location.frame,
2310  "end_frame", starting_frame_number - 1,
2311  "audio_pts_seconds", audio_pts_seconds);
2312 
2313 }
2314 
2315 
2316 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
2317 void FFmpegReader::Seek(int64_t requested_frame) {
2318  // Adjust for a requested frame that is too small or too large
2319  if (requested_frame < 1)
2320  requested_frame = 1;
2321  if (requested_frame > info.video_length)
2322  requested_frame = info.video_length;
2323  if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
2324  // Not possible to search past largest_frame once EOF is reached (no more packets)
2325  return;
2326  }
2327 
2328  // Debug output
2329  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
2330  "requested_frame", requested_frame,
2331  "seek_count", seek_count,
2332  "last_frame", last_frame);
2333 
2334  // Clear working cache (since we are seeking to another location in the file)
2335  working_cache.Clear();
2336 
2337  // Reset the last frame variable
2338  video_pts = 0.0;
2339  video_pts_seconds = NO_PTS_OFFSET;
2340  audio_pts = 0.0;
2341  audio_pts_seconds = NO_PTS_OFFSET;
2342  hold_packet = false;
2343  last_frame = 0;
2344  current_video_frame = 0;
2345  largest_frame_processed = 0;
2346  last_final_video_frame.reset();
2347  bool has_audio_override = info.has_audio;
2348  bool has_video_override = info.has_video;
2349 
2350  // Init end-of-file detection variables
2351  packet_status.reset(false);
2352 
2353  // Increment seek count
2354  seek_count++;
2355 
2356  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
2357  int buffer_amount = 12;
2358  if (requested_frame - buffer_amount < 20) {
2359  // prevent Open() from seeking again
2360  is_seeking = true;
2361 
2362  // Close and re-open file (basically seeking to frame 1)
2363  Close();
2364  Open();
2365 
2366  // Update overrides (since closing and re-opening might update these)
2367  info.has_audio = has_audio_override;
2368  info.has_video = has_video_override;
2369 
2370  // Not actually seeking, so clear these flags
2371  is_seeking = false;
2372  if (seek_count == 1) {
2373  // Don't redefine this on multiple seek attempts for a specific frame
2374  seeking_frame = 1;
2375  seeking_pts = ConvertFrameToVideoPTS(1);
2376  }
2377  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2378  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2379 
2380  } else {
2381  // Seek to nearest key-frame (aka, i-frame)
2382  bool seek_worked = false;
2383  int64_t seek_target = 0;
2384 
2385  // Seek video stream (if any), except album arts
2386  if (!seek_worked && info.has_video && !HasAlbumArt()) {
2387  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
2388  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2389  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking video stream");
2390  } else {
2391  // VIDEO SEEK
2392  is_video_seek = true;
2393  seek_worked = true;
2394  }
2395  }
2396 
2397  // Seek audio stream (if not already seeked... and if an audio stream is found)
2398  if (!seek_worked && info.has_audio) {
2399  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
2400  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2401  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking audio stream");
2402  } else {
2403  // AUDIO SEEK
2404  is_video_seek = false;
2405  seek_worked = true;
2406  }
2407  }
2408 
2409  // Was the seek successful?
2410  if (seek_worked) {
2411  // Flush audio buffer
2412  if (info.has_audio)
2413  avcodec_flush_buffers(aCodecCtx);
2414 
2415  // Flush video buffer
2416  if (info.has_video)
2417  avcodec_flush_buffers(pCodecCtx);
2418 
2419  // Reset previous audio location to zero
2420  previous_packet_location.frame = -1;
2421  previous_packet_location.sample_start = 0;
2422 
2423  // init seek flags
2424  is_seeking = true;
2425  if (seek_count == 1) {
2426  // Don't redefine this on multiple seek attempts for a specific frame
2427  seeking_pts = seek_target;
2428  seeking_frame = requested_frame;
2429  }
2430  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2431  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2432 
2433  } else {
2434  // seek failed
2435  seeking_pts = 0;
2436  seeking_frame = 0;
2437 
2438  // prevent Open() from seeking again
2439  is_seeking = true;
2440 
2441  // Close and re-open file (basically seeking to frame 1)
2442  Close();
2443  Open();
2444 
2445  // Not actually seeking, so clear these flags
2446  is_seeking = false;
2447 
2448  // disable seeking for this reader (since it failed)
2449  enable_seek = false;
2450 
2451  // Update overrides (since closing and re-opening might update these)
2452  info.has_audio = has_audio_override;
2453  info.has_video = has_video_override;
2454  }
2455  }
2456 }
2457 
2458 // Get the PTS for the current video packet
2459 int64_t FFmpegReader::GetPacketPTS() {
2460  if (packet) {
2461  int64_t current_pts = packet->pts;
2462  if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
2463  current_pts = packet->dts;
2464 
2465  // Return adjusted PTS
2466  return current_pts;
2467  } else {
2468  // No packet, return NO PTS
2469  return AV_NOPTS_VALUE;
2470  }
2471 }
2472 
2473 // Update PTS Offset (if any)
2474 void FFmpegReader::UpdatePTSOffset() {
2475  if (pts_offset_seconds != NO_PTS_OFFSET) {
2476  // Skip this method if we have already set PTS offset
2477  return;
2478  }
2479  pts_offset_seconds = 0.0;
2480  double video_pts_offset_seconds = 0.0;
2481  double audio_pts_offset_seconds = 0.0;
2482 
2483  bool has_video_pts = false;
2484  if (!info.has_video) {
2485  // Mark as checked
2486  has_video_pts = true;
2487  }
2488  bool has_audio_pts = false;
2489  if (!info.has_audio) {
2490  // Mark as checked
2491  has_audio_pts = true;
2492  }
2493 
2494  // Loop through the stream (until a packet from all streams is found)
2495  while (!has_video_pts || !has_audio_pts) {
2496  // Get the next packet (if any)
2497  if (GetNextPacket() < 0)
2498  // Break loop when no more packets found
2499  break;
2500 
2501  // Get PTS of this packet
2502  int64_t pts = GetPacketPTS();
2503 
2504  // Video packet
2505  if (!has_video_pts && packet->stream_index == videoStream) {
2506  // Get the video packet start time (in seconds)
2507  video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble());
2508 
2509  // Is timestamp close to zero (within X seconds)
2510  // Ignore wildly invalid timestamps (i.e. -234923423423)
2511  if (std::abs(video_pts_offset_seconds) <= 10.0) {
2512  has_video_pts = true;
2513  }
2514  }
2515  else if (!has_audio_pts && packet->stream_index == audioStream) {
2516  // Get the audio packet start time (in seconds)
2517  audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
2518 
2519  // Is timestamp close to zero (within X seconds)
2520  // Ignore wildly invalid timestamps (i.e. -234923423423)
2521  if (std::abs(audio_pts_offset_seconds) <= 10.0) {
2522  has_audio_pts = true;
2523  }
2524  }
2525  }
2526 
2527  // Choose timestamp origin:
2528  // - If video exists, anchor timeline frame mapping to video start.
2529  // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2.
2530  // - If no video exists (audio-only readers), use audio start.
2531  if (info.has_video && has_video_pts) {
2532  pts_offset_seconds = video_pts_offset_seconds;
2533  } else if (!info.has_video && has_audio_pts) {
2534  pts_offset_seconds = audio_pts_offset_seconds;
2535  } else if (has_video_pts && has_audio_pts) {
2536  // Fallback when stream flags are unusual but both timestamps exist.
2537  pts_offset_seconds = video_pts_offset_seconds;
2538  }
2539 }
2540 
2541 // Convert PTS into Frame Number
2542 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
2543  // Apply PTS offset
2544  int64_t previous_video_frame = current_video_frame;
2545  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2546  const double video_timebase_value =
2547  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2548  ? info.video_timebase.ToDouble()
2549  : (1.0 / 30.0);
2550 
2551  // Get the video packet start time (in seconds)
2552  double video_seconds = (double(pts) * video_timebase_value) + pts_offset_seconds;
2553 
2554  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2555  int64_t frame = round(video_seconds * fps_value) + 1;
2556 
2557  // Keep track of the expected video frame #
2558  if (current_video_frame == 0)
2559  current_video_frame = frame;
2560  else {
2561 
2562  // Sometimes frames are duplicated due to identical (or similar) timestamps
2563  if (frame == previous_video_frame) {
2564  // return -1 frame number
2565  frame = -1;
2566  } else {
2567  // Increment expected frame
2568  current_video_frame++;
2569  }
2570  }
2571 
2572  // Return frame #
2573  return frame;
2574 }
2575 
2576 // Convert Frame Number into Video PTS
2577 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2578  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2579  const double video_timebase_value =
2580  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2581  ? info.video_timebase.ToDouble()
2582  : (1.0 / 30.0);
2583 
2584  // Get timestamp of this frame (in seconds)
2585  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2586 
2587  // Calculate the # of video packets in this timestamp
2588  int64_t video_pts = round(seconds / video_timebase_value);
2589 
2590  // Apply PTS offset (opposite)
2591  return video_pts;
2592 }
2593 
2594 // Convert Frame Number into Video PTS
2595 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2596  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2597  const double audio_timebase_value =
2598  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2599  ? info.audio_timebase.ToDouble()
2600  : (1.0 / 48000.0);
2601 
2602  // Get timestamp of this frame (in seconds)
2603  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2604 
2605  // Calculate the # of audio packets in this timestamp
2606  int64_t audio_pts = round(seconds / audio_timebase_value);
2607 
2608  // Apply PTS offset (opposite)
2609  return audio_pts;
2610 }
2611 
2612 // Calculate Starting video frame and sample # for an audio PTS
2613 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2614  const double audio_timebase_value =
2615  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2616  ? info.audio_timebase.ToDouble()
2617  : (1.0 / 48000.0);
2618  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2619 
2620  // Get the audio packet start time (in seconds)
2621  double audio_seconds = (double(pts) * audio_timebase_value) + pts_offset_seconds;
2622 
2623  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2624  double frame = (audio_seconds * fps_value) + 1;
2625 
2626  // Frame # as a whole number (no more decimals)
2627  int64_t whole_frame = int64_t(frame);
2628 
2629  // Remove the whole number, and only get the decimal of the frame
2630  double sample_start_percentage = frame - double(whole_frame);
2631 
2632  // Get Samples per frame
2633  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2634 
2635  // Calculate the sample # to start on
2636  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2637 
2638  // Protect against broken (i.e. negative) timestamps
2639  if (whole_frame < 1)
2640  whole_frame = 1;
2641  if (sample_start < 0)
2642  sample_start = 0;
2643 
2644  // Prepare final audio packet location
2645  AudioLocation location = {whole_frame, sample_start};
2646 
2647  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2648  if (previous_packet_location.frame != -1) {
2649  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2650  int64_t orig_frame = location.frame;
2651  int orig_start = location.sample_start;
2652 
2653  // Update sample start, to prevent gaps in audio
2654  location.sample_start = previous_packet_location.sample_start;
2655  location.frame = previous_packet_location.frame;
2656 
2657  // Debug output
2658  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2659 
2660  } else {
2661  // Debug output
2662  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2663  }
2664  }
2665 
2666  // Set previous location
2667  previous_packet_location = location;
2668 
2669  // Return the associated video frame and starting sample #
2670  return location;
2671 }
2672 
2673 // Create a new Frame (or return an existing one) and add it to the working queue.
2674 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2675  // Check working cache
2676  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2677 
2678  if (!output) {
2679  // (re-)Check working cache
2680  output = working_cache.GetFrame(requested_frame);
2681  if(output) return output;
2682 
2683  // Create a new frame on the working cache
2684  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2685  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2686  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2687  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2688 
2689  working_cache.Add(output);
2690 
2691  // Set the largest processed frame (if this is larger)
2692  if (requested_frame > largest_frame_processed)
2693  largest_frame_processed = requested_frame;
2694  }
2695  // Return frame
2696  return output;
2697 }
2698 
2699 // Determine if frame is partial due to seek
2700 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2701 
2702  // Sometimes a seek gets partial frames, and we need to remove them
2703  bool seek_trash = false;
2704  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2705  if (seek_video_frame_found > max_seeked_frame) {
2706  max_seeked_frame = seek_video_frame_found;
2707  }
2708  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2709  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2710  seek_trash = true;
2711  }
2712 
2713  return seek_trash;
2714 }
2715 
2716 // Check the working queue, and move finished frames to the finished queue
2717 void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2718 
2719  // Prevent async calls to the following code
2720  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2721 
2722  // Get a list of current working queue frames in the cache (in-progress frames)
2723  std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2724  std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2725 
2726  // Loop through all working queue frames (sorted by frame #)
2727  for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2728  {
2729  // Get working frame
2730  std::shared_ptr<Frame> f = *working_itr;
2731 
2732  // Was a frame found? Is frame requested yet?
2733  if (!f || f->number > requested_frame) {
2734  // If not, skip to next one
2735  continue;
2736  }
2737 
2738  // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2739  double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2740  double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2741 
2742  // Determine if video and audio are ready (based on timestamps)
2743  bool is_video_ready = false;
2744  bool is_audio_ready = false;
2745  double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2746  if ((frame_pts_seconds <= video_pts_seconds)
2747  || (recent_pts_diff > 1.5)
2748  || packet_status.video_eof || packet_status.end_of_file) {
2749  // Video stream is past this frame (so it must be done)
2750  // OR video stream is too far behind, missing, or end-of-file
2751  is_video_ready = true;
2752  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2753  "frame_number", f->number,
2754  "frame_pts_seconds", frame_pts_seconds,
2755  "video_pts_seconds", video_pts_seconds,
2756  "recent_pts_diff", recent_pts_diff);
2757  if (info.has_video && !f->has_image_data &&
2758  (packet_status.video_eof || packet_status.end_of_file)) {
2759  // Frame has no image data. Prefer timeline-previous frames to preserve
2760  // visual order, especially when decode/prefetch is out-of-order.
2761  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2762  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2763  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2764  }
2765 
2766  // Fall back to last finalized timeline image (survives cache churn).
2767  if (!f->has_image_data
2768  && last_final_video_frame
2769  && last_final_video_frame->has_image_data
2770  && last_final_video_frame->number <= f->number) {
2771  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2772  }
2773 
2774  // Fall back to the last decoded image only when it is not from the future.
2775  if (!f->has_image_data
2776  && last_video_frame
2777  && last_video_frame->has_image_data
2778  && last_video_frame->number <= f->number) {
2779  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2780  }
2781 
2782  // Last-resort fallback if no prior image is available.
2783  if (!f->has_image_data) {
2784  ZmqLogger::Instance()->AppendDebugMethod(
2785  "FFmpegReader::CheckWorkingFrames (no previous image found; using black frame)",
2786  "frame_number", f->number);
2787  f->AddColor("#000000");
2788  }
2789  }
2790  }
2791 
2792  double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2793  if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2794  || (recent_pts_diff > 1.5)
2795  || packet_status.audio_eof || packet_status.end_of_file) {
2796  // Audio stream is past this frame (so it must be done)
2797  // OR audio stream is too far behind, missing, or end-of-file
2798  // Adding a bit of margin here, to allow for partial audio packets
2799  is_audio_ready = true;
2800  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2801  "frame_number", f->number,
2802  "frame_pts_seconds", frame_pts_seconds,
2803  "audio_pts_seconds", audio_pts_seconds,
2804  "audio_pts_diff", audio_pts_diff,
2805  "recent_pts_diff", recent_pts_diff);
2806  }
2807  bool is_seek_trash = IsPartialFrame(f->number);
2808 
2809  // Adjust for available streams
2810  if (!info.has_video) is_video_ready = true;
2811  if (!info.has_audio) is_audio_ready = true;
2812 
2813  // Debug output
2814  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2815  "frame_number", f->number,
2816  "is_video_ready", is_video_ready,
2817  "is_audio_ready", is_audio_ready,
2818  "video_eof", packet_status.video_eof,
2819  "audio_eof", packet_status.audio_eof,
2820  "end_of_file", packet_status.end_of_file);
2821 
2822  // Check if working frame is final
2823  if (info.has_video && !f->has_image_data
2824  && !packet_status.end_of_file && !is_seek_trash) {
2825  if (info.has_single_image) {
2826  // For still-image video (including attached cover art), reuse the most
2827  // recent image so playback does not stall waiting for video EOF.
2828  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2829  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2830  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2831  }
2832  if (!f->has_image_data
2833  && last_final_video_frame
2834  && last_final_video_frame->has_image_data
2835  && last_final_video_frame->number <= f->number) {
2836  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2837  }
2838  if (!f->has_image_data
2839  && last_video_frame
2840  && last_video_frame->has_image_data
2841  && last_video_frame->number <= f->number) {
2842  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2843  }
2844  }
2845 
2846  // If both streams have advanced past this frame but the decoder never
2847  // produced image data for it, reuse the most recent non-future image.
2848  // This avoids stalling indefinitely on sparse/missing decoded frames.
2849  if (!f->has_image_data && is_video_ready && is_audio_ready) {
2850  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2851  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2852  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2853  }
2854  if (!f->has_image_data
2855  && last_final_video_frame
2856  && last_final_video_frame->has_image_data
2857  && last_final_video_frame->number <= f->number) {
2858  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2859  }
2860  if (!f->has_image_data
2861  && last_video_frame
2862  && last_video_frame->has_image_data
2863  && last_video_frame->number <= f->number) {
2864  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2865  }
2866  }
2867 
2868  // Do not finalize non-EOF video frames without decoded image data.
2869  // This prevents repeated previous-frame fallbacks being cached as real frames.
2870  if (!f->has_image_data) {
2871  continue;
2872  }
2873  }
2874  if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2875  // Debug output
2876  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2877  "requested_frame", requested_frame,
2878  "f->number", f->number,
2879  "is_seek_trash", is_seek_trash,
2880  "Working Cache Count", working_cache.Count(),
2881  "Final Cache Count", final_cache.Count(),
2882  "end_of_file", packet_status.end_of_file);
2883 
2884  if (!is_seek_trash) {
2885  // Move frame to final cache
2886  final_cache.Add(f);
2887  if (f->has_image_data) {
2888  last_final_video_frame = f;
2889  }
2890 
2891  // Remove frame from working cache
2892  working_cache.Remove(f->number);
2893 
2894  // Update last frame processed
2895  last_frame = f->number;
2896  } else {
2897  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2898  working_cache.Remove(f->number);
2899  }
2900 
2901  }
2902  }
2903 
2904  // Clear vector of frames
2905  working_frames.clear();
2906  working_frames.shrink_to_fit();
2907 }
2908 
2909 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2910 void FFmpegReader::CheckFPS() {
2911  if (check_fps) {
2912  // Do not check FPS more than 1 time
2913  return;
2914  } else {
2915  check_fps = true;
2916  }
2917 
2918  int frames_per_second[3] = {0,0,0};
2919  int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
2920  int fps_index = 0;
2921 
2922  int all_frames_detected = 0;
2923  int starting_frames_detected = 0;
2924 
2925  // Loop through the stream
2926  while (true) {
2927  // Get the next packet (if any)
2928  if (GetNextPacket() < 0)
2929  // Break loop when no more packets found
2930  break;
2931 
2932  // Video packet
2933  if (packet->stream_index == videoStream) {
2934  // Get the video packet start time (in seconds)
2935  double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2936  fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
2937 
2938  // Is this video packet from the first few seconds?
2939  if (fps_index >= 0 && fps_index < max_fps_index) {
2940  // Yes, keep track of how many frames per second (over the first few seconds)
2941  starting_frames_detected++;
2942  frames_per_second[fps_index]++;
2943  }
2944 
2945  // Track all video packets detected
2946  all_frames_detected++;
2947  }
2948  }
2949 
2950  // Calculate FPS (based on the first few seconds of video packets)
2951  float avg_fps = 30.0;
2952  if (starting_frames_detected > 0 && fps_index > 0) {
2953  avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
2954  }
2955 
2956  // Verify average FPS is a reasonable value
2957  if (avg_fps < 8.0) {
2958  // Invalid FPS assumed, so switching to a sane default FPS instead
2959  avg_fps = 30.0;
2960  }
2961 
2962  // Update FPS (truncate average FPS to Integer)
2963  info.fps = Fraction(int(avg_fps), 1);
2964 
2965  // Update Duration and Length
2966  if (all_frames_detected > 0) {
2967  // Use all video frames detected to calculate # of frames
2968  info.video_length = all_frames_detected;
2969  info.duration = all_frames_detected / avg_fps;
2970  } else {
2971  // Use previous duration to calculate # of frames
2972  info.video_length = info.duration * avg_fps;
2973  }
2974 
2975  // Update video bit rate
2976  info.video_bit_rate = info.file_size / info.duration;
2977 }
2978 
2979 // Remove AVFrame from cache (and deallocate its memory)
2980 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2981  // Remove pFrame (if exists)
2982  if (remove_frame) {
2983  // Free memory
2984  av_freep(&remove_frame->data[0]);
2985 #ifndef WIN32
2986  AV_FREE_FRAME(&remove_frame);
2987 #endif
2988  }
2989 }
2990 
2991 // Remove AVPacket from cache (and deallocate its memory)
2992 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2993  // deallocate memory for packet
2994  AV_FREE_PACKET(remove_packet);
2995 
2996  // Delete the object
2997  delete remove_packet;
2998 }
2999 
3000 // Generate JSON string of this object
3001 std::string FFmpegReader::Json() const {
3002 
3003  // Return formatted string
3004  return JsonValue().toStyledString();
3005 }
3006 
3007 // Generate Json::Value for this object
3008 Json::Value FFmpegReader::JsonValue() const {
3009 
3010  // Create root json object
3011  Json::Value root = ReaderBase::JsonValue(); // get parent properties
3012  root["type"] = "FFmpegReader";
3013  root["path"] = path;
3014  switch (duration_strategy) {
3015  case DurationStrategy::VideoPreferred:
3016  root["duration_strategy"] = "VideoPreferred";
3017  break;
3018  case DurationStrategy::AudioPreferred:
3019  root["duration_strategy"] = "AudioPreferred";
3020  break;
3021  case DurationStrategy::LongestStream:
3022  default:
3023  root["duration_strategy"] = "LongestStream";
3024  break;
3025  }
3026 
3027  // return JsonValue
3028  return root;
3029 }
3030 
3031 // Load JSON string into this object
3032 void FFmpegReader::SetJson(const std::string value) {
3033 
3034  // Parse JSON string into JSON objects
3035  try {
3036  const Json::Value root = openshot::stringToJson(value);
3037  // Set all values that match
3038  SetJsonValue(root);
3039  }
3040  catch (const std::exception& e) {
3041  // Error parsing JSON (or missing keys)
3042  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
3043  }
3044 }
3045 
3046 // Load Json::Value into this object
3047 void FFmpegReader::SetJsonValue(const Json::Value root) {
3048 
3049  // Set parent data
3050  ReaderBase::SetJsonValue(root);
3051 
3052  // Set data from Json (if key is found)
3053  if (!root["path"].isNull())
3054  path = root["path"].asString();
3055  if (!root["duration_strategy"].isNull()) {
3056  const std::string strategy = root["duration_strategy"].asString();
3057  if (strategy == "VideoPreferred") {
3058  duration_strategy = DurationStrategy::VideoPreferred;
3059  } else if (strategy == "AudioPreferred") {
3060  duration_strategy = DurationStrategy::AudioPreferred;
3061  } else {
3062  duration_strategy = DurationStrategy::LongestStream;
3063  }
3064  }
3065 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::CacheMemory::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheMemory.cpp:224
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:318
openshot::ReaderInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
openshot::FFmpegReader::FFmpegReader
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Definition: FFmpegReader.cpp:102
openshot::Fraction::ToFloat
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
openshot::Settings::HARDWARE_DECODER
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:62
openshot::Coordinate::Y
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
openshot::CacheMemory::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheMemory.cpp:240
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::ReaderBase::JsonValue
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:106
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:178
openshot::TimelineBase::preview_width
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
openshot::PacketStatus::reset
void reset(bool eof)
Definition: FFmpegReader.h:70
openshot::CacheMemory::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:84
openshot::FFmpegReader::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: FFmpegReader.cpp:1197
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:327
openshot::CacheMemory::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:47
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:310
openshot::ReaderBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:157
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:259
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
openshot::Point::co
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
openshot::Clip::scale_y
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:319
openshot::AudioLocation
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
openshot::AudioLocation::frame
int64_t frame
Definition: AudioLocation.h:26
openshot::ZmqLogger::Log
void Log(std::string message)
Log message to all subscribers of this logger (if any)
Definition: ZmqLogger.cpp:103
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::DurationStrategy::AudioPreferred
@ AudioPreferred
Prefer the audio stream's duration, fallback to video then container.
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
openshot::AudioLocation::sample_start
int sample_start
Definition: AudioLocation.h:27
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:314
MemoryTrim.h
Cross-platform helper to encourage returning freed memory to the OS.
openshot::Keyframe::GetMaxPoint
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
openshot::ReaderBase::info
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:88
openshot::ReaderInfo::interlaced_frame
bool interlaced_frame
Definition: ReaderBase.h:56
Timeline.h
Header file for Timeline class.
openshot::Clip::ParentTimeline
void ParentTimeline(openshot::TimelineBase *new_timeline) override
Set associated Timeline pointer.
Definition: Clip.cpp:447
openshot::FFmpegReader::~FFmpegReader
virtual ~FFmpegReader()
Destructor.
Definition: FFmpegReader.cpp:137
openshot::ReaderInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
openshot::CacheMemory::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheMemory.cpp:158
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:315
openshot::ReaderInfo::duration
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::ReaderInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
openshot::FFmpegReader::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: FFmpegReader.cpp:3008
openshot::PacketStatus::audio_read
int64_t audio_read
Definition: FFmpegReader.h:51
openshot::ReaderInfo::width
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
openshot::FFmpegReader::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: FFmpegReader.cpp:3032
openshot::PacketStatus::packets_eof
bool packets_eof
Definition: FFmpegReader.h:57
hw_de_av_pix_fmt_global
AVPixelFormat hw_de_av_pix_fmt_global
Definition: FFmpegReader.cpp:72
openshot::PacketStatus::audio_decoded
int64_t audio_decoded
Definition: FFmpegReader.h:52
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::PacketStatus::video_read
int64_t video_read
Definition: FFmpegReader.h:49
hw_de_on
int hw_de_on
Definition: FFmpegReader.cpp:70
openshot::CacheBase::SetMaxBytesFromInfo
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:28
AV_ALLOCATE_IMAGE
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
Definition: FFmpegUtilities.h:311
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
openshot::Clip::scale_x
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:318
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:322
openshot::ReaderInfo::video_length
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
hw_de_av_device_type_global
AVHWDeviceType hw_de_av_device_type_global
Definition: FFmpegReader.cpp:73
openshot::ReaderInfo::height
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
openshot::PacketStatus::video_eof
bool video_eof
Definition: FFmpegReader.h:55
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
if
if(!codec) codec
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
OPEN_MP_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Definition: OpenMPUtilities.h:23
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:313
openshot::AudioLocation::is_near
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Definition: FFmpegReader.cpp:144
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:262
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::ReaderInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Settings::DE_LIMIT_HEIGHT_MAX
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:77
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:223
openshot::FFmpegReader::enable_seek
bool enable_seek
Definition: FFmpegReader.h:261
openshot::ReaderInfo::file_size
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:153
openshot::FFmpegReader::Open
void Open() override
Open File - which is called by the constructor automatically.
Definition: FFmpegReader.cpp:261
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:354
openshot::SCALE_CROP
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:264
SWRCONTEXT
#define SWRCONTEXT
Definition: FFmpegUtilities.h:265
openshot::PacketStatus::audio_eof
bool audio_eof
Definition: FFmpegReader.h:56
openshot::ReaderInfo::has_single_image
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
openshot::FFmpegReader::final_cache
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:257
openshot::ReaderInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:23
CropHelpers.h
Shared helpers for Crop effect scaling logic.
openshot::ReaderInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
openshot::DurationStrategy::LongestStream
@ LongestStream
Use the longest value from video, audio, or container.
openshot::FFmpegReader
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:103
path
path
Definition: FFmpegWriter.cpp:1474
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:193
openshot::ReaderInfo::audio_stream_index
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::DurationStrategy
DurationStrategy
This enumeration determines which duration source to favor.
Definition: Enums.h:60
openshot::ReaderInfo::audio_timebase
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
openshot::FFmpegReader::Close
void Close() override
Close File.
Definition: FFmpegReader.cpp:748
openshot::SCALE_FIT
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
openshot::PacketStatus::packets_read
int64_t packets_read()
Definition: FFmpegReader.h:60
openshot::ReaderInfo::pixel_format
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::ReaderInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::PacketStatus::packets_decoded
int64_t packets_decoded()
Definition: FFmpegReader.h:65
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:317
openshot::ReaderClosed
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:369
openshot::ReaderInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:316
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:110
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:323
AVCODEC_REGISTER_ALL
#define AVCODEC_REGISTER_ALL
Definition: FFmpegUtilities.h:306
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:263
openshot::Settings::DE_LIMIT_WIDTH_MAX
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:80
openshot::ReaderInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
AV_GET_SAMPLE_FORMAT
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:325
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
openshot::ReaderInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
openshot::PacketStatus::end_of_file
bool end_of_file
Definition: FFmpegReader.h:58
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::Clip::scale
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:179
openshot::ReaderInfo::top_field_first
bool top_field_first
Definition: ReaderBase.h:57
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:163
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:261
openshot::ReaderInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:305
openshot::DurationStrategy::VideoPreferred
@ VideoPreferred
Prefer the video stream's duration, fallback to audio then container.
openshot::CacheMemory::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:100
AV_GET_CODEC_CONTEXT
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:319
openshot::ReaderInfo::video_stream_index
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
openshot::FFmpegReader::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: FFmpegReader.cpp:3047
openshot::SCALE_STRETCH
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
openshot::ReaderInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:291
openshot::ReaderInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
openshot::ReaderInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::FFmpegReader::Json
std::string Json() const override
Generate JSON string of this object.
Definition: FFmpegReader.cpp:3001
openshot::FFmpegReader::HardwareDecodeSuccessful
bool HardwareDecodeSuccessful() const override
Return true if hardware decode was requested and successfully produced at least one frame.
Definition: FFmpegReader.cpp:1761
openshot::FFmpegReader::GetIsDurationKnown
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
Definition: FFmpegReader.cpp:1193
openshot::ApplyCropResizeScale
void ApplyCropResizeScale(Clip *clip, int source_width, int source_height, int &max_width, int &max_height)
Scale the requested max_width / max_height based on the Crop resize amount, capped by source size.
Definition: CropHelpers.cpp:40
openshot::PacketStatus::video_decoded
int64_t video_decoded
Definition: FFmpegReader.h:50
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1485
Exceptions.h
Header file for all Exception classes.
openshot::Settings::HW_DE_DEVICE_SET
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:83
FFmpegReader.h
Header file for FFmpegReader class.
openshot::ReaderBase::getFrameMutex
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ReaderBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:240
Generated by   doxygen 1.8.17