// Copyright Epic Games, Inc. All Rights Reserved. #include "MediaPlayerFacade.h" #include "MediaUtilsPrivate.h" #include "HAL/PlatformMath.h" #include "HAL/PlatformProcess.h" #include "IMediaCache.h" #include "IMediaControls.h" #include "IMediaModule.h" #include "IMediaOptions.h" #include "IMediaPlayer.h" #include "IMediaPlayerFactory.h" #include "IMediaSamples.h" #include "IMediaAudioSample.h" #include "IMediaTextureSample.h" #include "IMediaOverlaySample.h" #include "IMediaTracks.h" #include "IMediaView.h" #include "IMediaTicker.h" #include "MediaPlayerOptions.h" #include "Math/NumericLimits.h" #include "Misc/CoreMisc.h" #include "Misc/ScopeLock.h" #include "Modules/ModuleManager.h" #include "MediaHelpers.h" #include "MediaSampleCache.h" #include "MediaSampleQueueDepths.h" #include "MediaSampleQueue.h" #include "Async/Async.h" #include #define MEDIAPLAYERFACADE_DISABLE_BLOCKING 0 #define MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS 0 /** Time spent in media player facade closing media. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade Close"), STAT_MediaUtils_FacadeClose, STATGROUP_Media); /** Time spent in media player facade opening media. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade Open"), STAT_MediaUtils_FacadeOpen, STATGROUP_Media); /** Time spent in media player facade event processing. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade ProcessEvent"), STAT_MediaUtils_FacadeProcessEvent, STATGROUP_Media); /** Time spent in media player facade fetch tick. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade TickFetch"), STAT_MediaUtils_FacadeTickFetch, STATGROUP_Media); /** Time spent in media player facade input tick. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade TickInput"), STAT_MediaUtils_FacadeTickInput, STATGROUP_Media); /** Time spent in media player facade output tick. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade TickOutput"), STAT_MediaUtils_FacadeTickOutput, STATGROUP_Media); /** Time spent in media player facade high frequency tick. */ DECLARE_CYCLE_STAT(TEXT("MediaUtils MediaPlayerFacade TickTickable"), STAT_MediaUtils_FacadeTickTickable, STATGROUP_Media); /** Player time on main thread during last fetch tick. */ DECLARE_FLOAT_COUNTER_STAT(TEXT("MediaPlayerFacade PlaybackTime"), STAT_MediaUtils_FacadeTime, STATGROUP_Media); /** Number of video samples currently in the queue. */ DECLARE_DWORD_COUNTER_STAT(TEXT("MediaPlayerFacade NumVideoSamples"), STAT_MediaUtils_FacadeNumVideoSamples, STATGROUP_Media); /** Number of audio samples currently in the queue. */ DECLARE_DWORD_COUNTER_STAT(TEXT("MediaPlayerFacade NumAudioSamples"), STAT_MediaUtils_FacadeNumAudioSamples, STATGROUP_Media); /** Number of purged video samples */ DECLARE_DWORD_COUNTER_STAT(TEXT("MediaPlayerFacade NumPurgedVideoSamples"), STAT_MediaUtils_FacadeNumPurgedVideoSamples, STATGROUP_Media); DECLARE_DWORD_ACCUMULATOR_STAT(TEXT("MediaPlayerFacade TotalPurgedVideoSamples"), STAT_MediaUtils_FacadeTotalPurgedVideoSamples, STATGROUP_Media); /** Number of purged subtitle samples */ DECLARE_DWORD_COUNTER_STAT(TEXT("MediaPlayerFacade NumPurgedSubtitleSamples"), STAT_MediaUtils_FacadeNumPurgedSubtitleSamples, STATGROUP_Media); DECLARE_DWORD_ACCUMULATOR_STAT(TEXT("MediaPlayerFacade TotalPurgedSubtitleSamples"), STAT_MediaUtils_FacadeTotalPurgedSubtitleSamples, STATGROUP_Media); /** Number of purged caption samples */ DECLARE_DWORD_COUNTER_STAT(TEXT("MediaPlayerFacade NumPurgedCaptionSamples"), STAT_MediaUtils_FacadeNumPurgedCaptionSamples, STATGROUP_Media); DECLARE_DWORD_ACCUMULATOR_STAT(TEXT("MediaPlayerFacade TotalPurgedCaptionSamples"), STAT_MediaUtils_FacadeTotalPurgedCaptionSamples, STATGROUP_Media); /* Some constants *****************************************************************************/ static const double kMaxTimeSinceFrameStart = 0.300; // max seconds we allow between the start of the frame and the player facade timing computations (to catch suspended apps & debugging) static const double kMaxTimeSinceAudioTimeSampling = 0.250; // max seconds we allow to have passed between the last audio timing sampling and the player facade timing computations (to catch suspended apps & debugging - some platforms do update audio at a farily low rate: hence the big tollerance) static const double kOutdatedVideoSamplesTolerance = 0.080; // seconds video samples are allowed to be "too old" to stay in the player's output queue despite of calculations indicating they need to go static const double kOutdatedSubtitleSamplesTolerance = 1.0; // seconds subtitle samples are allowed to be "too old" to stay in the player's output queue despite of calculations indicating they need to go static const double kOutdatedSamplePurgeRange = 1.0; // milliseconds for pseudo DT timespan used with async purging of outdated video samples static const int32 kMinFramesInVideoQueueToPurge = 3; // we only consider purging any old frames from the video queue if more than these are present (to not kill a slow playback entirely) static const int32 kMinFramesInSubtitleQueueToPurge = 3; // we only consider purging any old frames from the subtitle queue if more than these are present (to not kill a slow playback entirely) static const int32 kMinFramesInCaptionQueueToPurge = 3; // we only consider purging any old frames from the caption queue if more than these are present (to not kill a slow playback entirely) /* Cvars *****************************************************************************/ namespace UE::MediaUtils::Private { #if !UE_BUILD_SHIPPING TAutoConsoleVariable CVarTestForcePlayerCreateFailed( TEXT("m.Test.ForcePlayerCreateFailed"), false, TEXT("Whether force media player creation to fail."), ECVF_ReadOnly ); #endif // !UE_BUILD_SHIPPING float GBlockOnFetchTimeout = 10.0f; static FAutoConsoleVariableRef CVarMediaUtilsBlockOnFetchTimeout( TEXT("MediaUtils.BlockOnFetchTimeout"), GBlockOnFetchTimeout, TEXT("Maximum time that TickInput/Fetch will block waiting for samples (in seconds).\n") ); } /* Local helpers *****************************************************************************/ namespace MediaPlayerFacade { const FTimespan AudioPreroll = FTimespan::FromSeconds(1.0); const FTimespan MetadataPreroll = FTimespan::FromSeconds(1.0); } static FTimespan WrappedModulo(FTimespan Time, FTimespan Duration) { return (Time >= FTimespan::Zero()) ? (Time % Duration) : (Duration + (Time % Duration)); } static bool IsDurationValidAndFinite(FTimespan Duration) { return (Duration != FTimespan::Zero() && Duration.GetTicks() != TNumericLimits::Max()); } /* FMediaPlayerFacade structors *****************************************************************************/ FMediaPlayerFacade::FMediaPlayerFacade(TWeakObjectPtr InMediaPlayer) : TimeDelay(FTimespan::Zero()) , BlockOnRange(this) , Cache(new FMediaSampleCache) , LastRate(0.0f) , CurrentRate(0.0f) , bHaveActiveAudio(false) , VideoSampleAvailability(-1) , AudioSampleAvailability(-1) , bAreEventsSafeForAnyThread(false) , MediaPlayer(InMediaPlayer) { BlockOnRangeDisabled = false; MediaModule = FModuleManager::LoadModulePtr("Media"); bDidRecentPlayerHaveError = false; ResetTracks(); } FMediaPlayerFacade::~FMediaPlayerFacade() { FMediaSampleSinkEventData Data; Data.Detached.MediaPlayer = MediaPlayer.Get(); SendSinkEvent(EMediaSampleSinkEvent::Detached, Data); if (Player.IsValid()) { { FScopeLock Lock(&CriticalSection); Player->Close(); } NotifyLifetimeManagerDelegate_PlayerClosed(); DestroyPlayer(); } delete Cache; Cache = nullptr; } /* FMediaPlayerFacade interface *****************************************************************************/ void FMediaPlayerFacade::AddAudioSampleSink(const TSharedRef& SampleSink) { FScopeLock Lock(&CriticalSection); AudioSampleSinks.Add(SampleSink); PrimaryAudioSink = AudioSampleSinks.GetPrimaryAudioSink(); } void FMediaPlayerFacade::AddCaptionSampleSink(const TSharedRef& SampleSink) { CaptionSampleSinks.Add(SampleSink); } void FMediaPlayerFacade::AddMetadataSampleSink(const TSharedRef& SampleSink) { FScopeLock Lock(&CriticalSection); MetadataSampleSinks.Add(SampleSink); } void FMediaPlayerFacade::AddSubtitleSampleSink(const TSharedRef& SampleSink) { SubtitleSampleSinks.Add(SampleSink); } void FMediaPlayerFacade::AddVideoSampleSink(const TSharedRef& SampleSink) { VideoSampleSinks.Add(SampleSink); } bool FMediaPlayerFacade::CanPause() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetControls().CanControl(EMediaControl::Pause); } bool FMediaPlayerFacade::CanPlayUrl(const FString& Url, const IMediaOptions* Options) { if (MediaModule == nullptr) { return false; } const FString RunningPlatformName(FPlatformProperties::IniPlatformName()); const TArray& PlayerFactories = MediaModule->GetPlayerFactories(); for (IMediaPlayerFactory* Factory : PlayerFactories) { if (Factory->SupportsPlatform(RunningPlatformName) && Factory->CanPlayUrl(Url, Options)) { return true; } } return false; } bool FMediaPlayerFacade::CanResume() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetControls().CanControl(EMediaControl::Resume); } bool FMediaPlayerFacade::CanScrub() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetControls().CanControl(EMediaControl::Scrub); } bool FMediaPlayerFacade::CanSeek() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetControls().CanControl(EMediaControl::Seek); } bool FMediaPlayerFacade::SupportsPlaybackTimeRange() const { TSharedPtr CurrentPlayer(Player); return CurrentPlayer.IsValid() ? CurrentPlayer->GetControls().CanControl(EMediaControl::PlaybackRange) : false; } void FMediaPlayerFacade::Close() { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeClose); if (CurrentUrl.IsEmpty()) { return; } TSharedPtr CurrentPlayer(Player); if (CurrentPlayer.IsValid()) { { FScopeLock Lock(&CriticalSection); CurrentPlayer->Close(); } NotifyLifetimeManagerDelegate_PlayerClosed(); } Flush(); ReInit(); BlockOnRange.Reset(); bDidRecentPlayerHaveError = false; } uint32 FMediaPlayerFacade::GetAudioTrackChannels(int32 TrackIndex, int32 FormatIndex) const { FMediaAudioTrackFormat Format; return GetAudioTrackFormat(TrackIndex, FormatIndex, Format) ? Format.NumChannels : 0; } uint32 FMediaPlayerFacade::GetAudioTrackSampleRate(int32 TrackIndex, int32 FormatIndex) const { FMediaAudioTrackFormat Format; return GetAudioTrackFormat(TrackIndex, FormatIndex, Format) ? Format.SampleRate : 0; } FString FMediaPlayerFacade::GetAudioTrackType(int32 TrackIndex, int32 FormatIndex) const { FMediaAudioTrackFormat Format; return GetAudioTrackFormat(TrackIndex, FormatIndex, Format) ? Format.TypeName : FString(); } FTimespan FMediaPlayerFacade::GetDuration() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FTimespan::Zero(); } return CurrentPlayer->GetControls().GetDuration(); } const FGuid& FMediaPlayerFacade::GetGuid() { return PlayerGuid; } FString FMediaPlayerFacade::GetInfo() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FString(); } return CurrentPlayer->GetInfo(); } FVariant FMediaPlayerFacade::GetMediaInfo(FName InfoName) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FVariant(); } return CurrentPlayer->GetMediaInfo(InfoName); } FText FMediaPlayerFacade::GetMediaName() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FText::GetEmpty(); } return CurrentPlayer->GetMediaName(); } TSharedPtr>>, ESPMode::ThreadSafe> FMediaPlayerFacade::GetMediaMetadata() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return nullptr; } return CurrentPlayer->GetMediaMetadata(); } int32 FMediaPlayerFacade::GetNumTracks(EMediaTrackType TrackType) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return 0; } return CurrentPlayer->GetTracks().GetNumTracks(TrackType); } int32 FMediaPlayerFacade::GetNumTrackFormats(EMediaTrackType TrackType, int32 TrackIndex) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return 0; } return CurrentPlayer->GetTracks().GetNumTrackFormats(TrackType, TrackIndex); } FName FMediaPlayerFacade::GetPlayerName() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return NAME_None; } return MediaModule->GetPlayerFactory(CurrentPlayer->GetPlayerPluginGUID())->GetPlayerName(); } float FMediaPlayerFacade::GetRate() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return 0.0f; } return CurrentPlayer->GetControls().GetRate(); } FString FMediaPlayerFacade::GetStats() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FString(); } return CurrentPlayer->GetStats(); } TRangeSet FMediaPlayerFacade::GetSupportedRates(bool Unthinned) const { const EMediaRateThinning Thinning = Unthinned ? EMediaRateThinning::Unthinned : EMediaRateThinning::Thinned; TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return TRangeSet(); } return CurrentPlayer->GetControls().GetSupportedRates(Thinning); } bool FMediaPlayerFacade::HaveVideoPlayback() const { return VideoSampleSinks.Num() && (GetSelectedTrack(EMediaTrackType::Video) != INDEX_NONE); } bool FMediaPlayerFacade::HaveAudioPlayback() const { return PrimaryAudioSink.IsValid() && (GetSelectedTrack(EMediaTrackType::Audio) != INDEX_NONE); } FTimespan FMediaPlayerFacade::GetTime() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FTimespan::Zero(); // no media opened } FTimespan Result; if (CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // New style: framework controls timing - we use GetTimeStamp() and return the legacy part of the value FMediaTimeStamp TimeStamp = GetTimeStamp(); return TimeStamp.IsValid() ? TimeStamp.Time : FTimespan::Zero(); } else { // Old style: ask the player for timing Result = CurrentPlayer->GetControls().GetTime() - TimeDelay; if (Result.GetTicks() < 0) { Result = FTimespan::Zero(); } } return Result; } FMediaTimeStamp FMediaPlayerFacade::GetTimeStamp() const { return GetTimeStampInternal(false); } FMediaTimeStamp FMediaPlayerFacade::GetDisplayTimeStamp() const { return GetTimeStampInternal(true); } TOptional FMediaPlayerFacade::GetVideoTimecode() const { FScopeLock Lock(&LastTimeValuesCS); return MostRecentlyDeliveredVideoFrameTimecode; } TRange FMediaPlayerFacade::GetLastProcessedVideoSampleTimeRange() const { FScopeLock Lock(&LastTimeValuesCS); return LastVideoSampleProcessedTimeRange; } FMediaTimeStamp FMediaPlayerFacade::GetTimeStampInternal(bool bForDisplay) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FMediaTimeStamp(); } FScopeLock Lock(&LastTimeValuesCS); if (!CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Make sure we can return values for V1 players... return FMediaTimeStamp(GetTime()); } // Check if the value is for display purposes. If so: do we seek right now? if (bForDisplay && SeekTargetTime.IsValid()) { return SeekTargetTime; } // Check if there are video samples present or presence is unknown. // Only when we know for sure that there are none because the existing video stream has ended do we set this to false. bool bHaveVideoSamples = VideoSampleAvailability != 0; if (HaveVideoPlayback() && bHaveVideoSamples) { /* Returning the precise time of the sample returned during TickFetch() */ return bForDisplay ? CurrentFrameVideoDisplayTimeStamp : CurrentFrameVideoTimeStamp; } else if (HaveAudioPlayback()) { /* We grab the last processed audio sample timestamp when it gets passed out to the sink(s) and keep it as "the value" for the frame (on the gamethread) -- an approximation, but better then having it return new values each time its called in one and the same frame... */ return CurrentFrameAudioTimeStamp; } // we assume video and/or audio to be present in any stream we play - otherwise: no time info // (at least for now) return FMediaTimeStamp(); } FText FMediaPlayerFacade::GetTrackDisplayName(EMediaTrackType TrackType, int32 TrackIndex) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FText::GetEmpty(); } return CurrentPlayer->GetTracks().GetTrackDisplayName((EMediaTrackType)TrackType, TrackIndex); } int32 FMediaPlayerFacade::GetTrackFormat(EMediaTrackType TrackType, int32 TrackIndex) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return INDEX_NONE; } return CurrentPlayer->GetTracks().GetTrackFormat((EMediaTrackType)TrackType, TrackIndex); } FString FMediaPlayerFacade::GetTrackLanguage(EMediaTrackType TrackType, int32 TrackIndex) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return FString(); } return CurrentPlayer->GetTracks().GetTrackLanguage((EMediaTrackType)TrackType, TrackIndex); } float FMediaPlayerFacade::GetVideoTrackAspectRatio(int32 TrackIndex, int32 FormatIndex) const { FMediaVideoTrackFormat Format; return (GetVideoTrackFormat(TrackIndex, FormatIndex, Format) && (Format.Dim.Y != 0)) ? ((float)(Format.Dim.X) / (float)Format.Dim.Y) : 0.0f; } FIntPoint FMediaPlayerFacade::GetVideoTrackDimensions(int32 TrackIndex, int32 FormatIndex) const { FMediaVideoTrackFormat Format; return GetVideoTrackFormat(TrackIndex, FormatIndex, Format) ? Format.Dim : FIntPoint::ZeroValue; } float FMediaPlayerFacade::GetVideoTrackFrameRate(int32 TrackIndex, int32 FormatIndex) const { FMediaVideoTrackFormat Format; return GetVideoTrackFormat(TrackIndex, FormatIndex, Format) ? Format.FrameRate : 0.0f; } TRange FMediaPlayerFacade::GetVideoTrackFrameRates(int32 TrackIndex, int32 FormatIndex) const { FMediaVideoTrackFormat Format; return GetVideoTrackFormat(TrackIndex, FormatIndex, Format) ? Format.FrameRates : TRange::Empty(); } FString FMediaPlayerFacade::GetVideoTrackType(int32 TrackIndex, int32 FormatIndex) const { FMediaVideoTrackFormat Format; return GetVideoTrackFormat(TrackIndex, FormatIndex, Format) ? Format.TypeName : FString(); } bool FMediaPlayerFacade::GetViewField(float& OutHorizontal, float& OutVertical) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetView().GetViewField(OutHorizontal, OutVertical); } bool FMediaPlayerFacade::GetViewOrientation(FQuat& OutOrientation) const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetView().GetViewOrientation(OutOrientation); } bool FMediaPlayerFacade::HasError() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return bDidRecentPlayerHaveError; } return (CurrentPlayer->GetControls().GetState() == EMediaState::Error); } bool FMediaPlayerFacade::IsBuffering() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return EnumHasAnyFlags(CurrentPlayer->GetControls().GetStatus(), EMediaStatus::Buffering); } bool FMediaPlayerFacade::IsConnecting() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return EnumHasAnyFlags(CurrentPlayer->GetControls().GetStatus(), EMediaStatus::Connecting); } bool FMediaPlayerFacade::IsLooping() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return CurrentPlayer->GetControls().IsLooping(); } bool FMediaPlayerFacade::IsPaused() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return (CurrentPlayer->GetControls().GetState() == EMediaState::Paused); } bool FMediaPlayerFacade::IsPlaying() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return (CurrentPlayer->GetControls().GetState() == EMediaState::Playing); } bool FMediaPlayerFacade::IsPreparing() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return (CurrentPlayer->GetControls().GetState() == EMediaState::Preparing); } bool FMediaPlayerFacade::IsClosed() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } return (CurrentPlayer->GetControls().GetState() == EMediaState::Closed); } bool FMediaPlayerFacade::IsReady() const { TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { return false; } EMediaState State = CurrentPlayer->GetControls().GetState(); return ((State != EMediaState::Closed) && (State != EMediaState::Error) && (State != EMediaState::Preparing)); } // ---------------------------------------------------------------------------------------------------------------------------------------------- class FMediaPlayerLifecycleManagerDelegateOpenRequest : public IMediaPlayerLifecycleManagerDelegate::IOpenRequest { public: FMediaPlayerLifecycleManagerDelegateOpenRequest(const FString& InUrl, const IMediaOptions* InOptions, const FMediaPlayerOptions* InPlayerOptions, IMediaPlayerFactory* InPlayerFactory, TSharedPtr InReusedPlayer, bool bInWillCreatePlayer, uint32 InWillUseNewResources) : Url(InUrl) , Options(InOptions) , OptionsObject(InOptions ? InOptions->ToUObject() : nullptr) , PlayerFactory(InPlayerFactory) , ReusedPlayer(InReusedPlayer) , bWillCreatePlayer(bInWillCreatePlayer) , NewResources(InWillUseNewResources) { if (InPlayerOptions) { PlayerOptions = *InPlayerOptions; } } virtual const FString& GetUrl() const override { return Url; } virtual const IMediaOptions* GetOptions() const override { return OptionsObject.IsStale() ? nullptr : Options; } virtual const FMediaPlayerOptions* GetPlayerOptions() const override { return PlayerOptions.IsSet() ? &PlayerOptions.GetValue() : nullptr; } virtual IMediaPlayerFactory* GetPlayerFactory() const override { return PlayerFactory; } virtual bool WillCreateNewPlayer() const { return bWillCreatePlayer; } virtual bool WillUseNewResources(uint32 ResourceFlags) const { return !!(NewResources & ResourceFlags); } const TSharedPtr& GetReusedPlayer() const { return ReusedPlayer; } private: FString Url; const IMediaOptions* Options; FWeakObjectPtr OptionsObject; TOptional PlayerOptions; IMediaPlayerFactory* PlayerFactory; TSharedPtr ReusedPlayer; bool bWillCreatePlayer; uint32 NewResources; }; class FMediaPlayerLifecycleManagerDelegateControl : public IMediaPlayerLifecycleManagerDelegate::IControl, public TSharedFromThis { public: FMediaPlayerLifecycleManagerDelegateControl(TWeakPtr InFacade) : Facade(InFacade), InstanceID(~0), SubmittedRequest(false) {} virtual ~FMediaPlayerLifecycleManagerDelegateControl() { if (!SubmittedRequest) { if (TSharedPtr PinnedFacade = Facade.Pin()) { PinnedFacade->ReceiveMediaEvent(EMediaEvent::MediaOpenFailed); } } } virtual bool SubmitOpenRequest(IMediaPlayerLifecycleManagerDelegate::IOpenRequestRef&& OpenRequest) override { if (TSharedPtr PinnedFacade = Facade.Pin()) { const FMediaPlayerLifecycleManagerDelegateOpenRequest* OR = static_cast(OpenRequest.Get()); if (PinnedFacade->ContinueOpen(AsShared(), OR->GetUrl(), OR->GetOptions(), OR->GetPlayerOptions(), OR->GetPlayerFactory(), OR->GetReusedPlayer(), OR->WillCreateNewPlayer(), InstanceID)) { SubmittedRequest = true; } //note: we return "true" in all cases in which we were able to get to call "ContinueOpen". Failures in here will be messaged to the delegate using the OnMediaPlayerCreateFailed() method // (returning true here allows for capturing an unlikely early death of the facade while protecting us from double-handling the failure of the creation in the delegate) return true; } return false; } virtual TSharedPtr GetFacade() const override { return Facade.Pin(); } virtual uint64 GetMediaPlayerInstanceID() const override { return InstanceID; } void SetInstanceID(uint64 InInstanceID) { InstanceID = InInstanceID; } void Reset() { SubmittedRequest = true; } private: TWeakPtr Facade; uint64 InstanceID; bool SubmittedRequest; }; // ---------------------------------------------------------------------------------------------------------------------------------------------- bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerOpen(IMediaPlayerLifecycleManagerDelegate::IControlRef& NewLifecycleManagerDelegateControl, const FString& Url, const IMediaOptions* Options, const FMediaPlayerOptions* PlayerOptions, IMediaPlayerFactory* PlayerFactory, bool bWillCreatePlayer, uint32 WillUseNewResources, uint64 NewPlayerInstanceID) { check(IsInGameThread() || IsInSlateThread()); if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { NewLifecycleManagerDelegateControl = MakeShared(AsShared()); if (NewLifecycleManagerDelegateControl.IsValid()) { // Set instance ID we will use for a new player if we get the go-ahead to create it (old ID if player is about to be reused) static_cast(NewLifecycleManagerDelegateControl.Get())->SetInstanceID(NewPlayerInstanceID); IMediaPlayerLifecycleManagerDelegate::IOpenRequestRef OpenRequest(new FMediaPlayerLifecycleManagerDelegateOpenRequest(Url, Options, PlayerOptions, PlayerFactory, !bWillCreatePlayer ? Player : TSharedPtr(), bWillCreatePlayer, WillUseNewResources)); if (OpenRequest.IsValid()) { if (Delegate->OnMediaPlayerOpen(NewLifecycleManagerDelegateControl, OpenRequest)) { return true; } } static_cast(NewLifecycleManagerDelegateControl.Get())->Reset(); } } return false; } bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerCreated() { check(IsInGameThread() || IsInSlateThread()); check(Player.IsValid()); if (LifecycleManagerDelegateControl.IsValid()) { if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { Delegate->OnMediaPlayerCreated(LifecycleManagerDelegateControl); return true; } } return false; } bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerCreateFailed() { check(IsInGameThread() || IsInSlateThread()); if (LifecycleManagerDelegateControl.IsValid()) { if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { Delegate->OnMediaPlayerCreateFailed(LifecycleManagerDelegateControl); return true; } } return false; } bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerClosed() { check(IsInGameThread() || IsInSlateThread()); if (LifecycleManagerDelegateControl.IsValid()) { if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { Delegate->OnMediaPlayerClosed(LifecycleManagerDelegateControl); return true; } } return false; } bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerDestroyed() { check(IsInGameThread() || IsInSlateThread()); if (LifecycleManagerDelegateControl.IsValid()) { if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { Delegate->OnMediaPlayerDestroyed(LifecycleManagerDelegateControl); return true; } } return false; } bool FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerResourcesReleased(uint32 ResourceFlags) { check(IsInGameThread() || IsInSlateThread()); if (LifecycleManagerDelegateControl.IsValid()) { if (IMediaPlayerLifecycleManagerDelegate* Delegate = MediaModule->GetPlayerLifecycleManagerDelegate()) { Delegate->OnMediaPlayerResourcesReleased(LifecycleManagerDelegateControl, ResourceFlags); return true; } } return false; } // ---------------------------------------------------------------------------------------------------------------------------------------------- void FMediaPlayerFacade::DestroyPlayer() { FScopeLock Lock(&CriticalSection); if (!Player.IsValid()) { return; } Player.Reset(); NotifyLifetimeManagerDelegate_PlayerDestroyed(); if (!PlayerUsesResourceReleaseNotification) { NotifyLifetimeManagerDelegate_PlayerResourcesReleased(IMediaPlayerLifecycleManagerDelegate::ResourceFlags_All); } } bool FMediaPlayerFacade::Open(const FString& Url, const IMediaOptions* Options, const FMediaPlayerOptions* PlayerOptions) { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeOpen); ActivePlayerOptions.Reset(); if (IsRunningDedicatedServer()) { return false; } Close(); if (Url.IsEmpty()) { return false; } check(MediaModule); // find a player factory for the intended playback IMediaPlayerFactory* PlayerFactory = GetPlayerFactoryForUrl(Url, Options); if (PlayerFactory == nullptr) { return false; } IMediaPlayerFactory* OldFactory(Player.IsValid() ? MediaModule->GetPlayerFactory(Player->GetPlayerPluginGUID()) : nullptr); bool bWillCreatePlayer = (!Player.IsValid() || PlayerFactory != OldFactory); uint64 NewPlayerInstanceID; uint32 WillUseNewResources; if (bWillCreatePlayer) { NewPlayerInstanceID = MediaModule->CreateMediaPlayerInstanceID(); WillUseNewResources = IMediaPlayerLifecycleManagerDelegate::ResourceFlags_All; // as we create a new player we assume all resources a newly created in any case } else { check(Player.IsValid()); NewPlayerInstanceID = PlayerInstanceID; WillUseNewResources = Player->GetNewResourcesOnOpen(); // ask player what resources it will create again even if it already exists } IMediaPlayerLifecycleManagerDelegate::IControlRef NewLifecycleManagerDelegateControl; if (FMediaPlayerFacade::NotifyLifetimeManagerDelegate_PlayerOpen(NewLifecycleManagerDelegateControl, Url, Options, PlayerOptions, PlayerFactory, bWillCreatePlayer, WillUseNewResources, NewPlayerInstanceID)) { // Assume all is well: the delegate will either (have) submit(ted) the request or not -- in any case we need to assume the best -> "true" return true; } // We did not notify successfully or the delegate will not submit the request in its own. Do so here... return ContinueOpen(NewLifecycleManagerDelegateControl, Url, Options, PlayerOptions, PlayerFactory, Player, bWillCreatePlayer, NewPlayerInstanceID); } bool FMediaPlayerFacade::ContinueOpen(IMediaPlayerLifecycleManagerDelegate::IControlRef NewLifecycleManagerDelegateControl, const FString& Url, const IMediaOptions* Options, const FMediaPlayerOptions* PlayerOptions, IMediaPlayerFactory* PlayerFactory, TSharedPtr ReusedPlayer, bool bCreateNewPlayer, uint64 NewPlayerInstanceID) { // Create or reuse player TSharedPtr NewPlayer(bCreateNewPlayer ? PlayerFactory->CreatePlayer(*this) : ReusedPlayer); // Continue initialization --------------------------------------- if (NewPlayer != Player) { DestroyPlayer(); class FAsyncResourceReleaseNotification : public IMediaPlayer::IAsyncResourceReleaseNotification { public: FAsyncResourceReleaseNotification(IMediaPlayerLifecycleManagerDelegate::IControlRef InDelegateControl) : DelegateControl(InDelegateControl) {} virtual void Signal(uint32 ResourceFlags) override { TFunction NotifyTask = [TargetDelegateControl = DelegateControl, ResourceFlags]() { // Get MediaModule & check if it is already unloaded... IMediaModule* TargetMediaModule = FModuleManager::GetModulePtr("Media"); if (TargetMediaModule) { // Delegate still there? if (IMediaPlayerLifecycleManagerDelegate* Delegate = TargetMediaModule->GetPlayerLifecycleManagerDelegate()) { // Notify it! Delegate->OnMediaPlayerResourcesReleased(TargetDelegateControl, ResourceFlags); } } }; Async(EAsyncExecution::TaskGraphMainThread, NotifyTask); }; IMediaModule* MediaModule; IMediaPlayerLifecycleManagerDelegate::IControlRef DelegateControl; }; FScopeLock Lock(&CriticalSection); Player = NewPlayer; PlayerInstanceID = NewPlayerInstanceID; LifecycleManagerDelegateControl = NewLifecycleManagerDelegateControl; PlayerUsesResourceReleaseNotification = LifecycleManagerDelegateControl.IsValid() ? Player->SetAsyncResourceReleaseNotification(TSharedRef(new FAsyncResourceReleaseNotification(LifecycleManagerDelegateControl))) : false; } else { LifecycleManagerDelegateControl = NewLifecycleManagerDelegateControl; } bool bIsRequestInvalid = (Player == nullptr); #if !UE_BUILD_SHIPPING bIsRequestInvalid = bIsRequestInvalid || UE::MediaUtils::Private::CVarTestForcePlayerCreateFailed.GetValueOnAnyThread(); #endif if (bIsRequestInvalid) { NotifyLifetimeManagerDelegate_PlayerCreateFailed(); // Make sure we don't get called from the "tickable" thread anymore - no need as we have no player MediaModule->GetTicker().RemoveTickable(AsShared()); return false; } // Make sure we get ticked on the "tickable" thread // (this will not re-add us, should we already be registered) MediaModule->GetTicker().AddTickable(AsShared()); // update the Guid Player->SetGuid(PlayerGuid); CurrentUrl = Url; if (PlayerOptions) { ActivePlayerOptions = *PlayerOptions; } // open the new media source if (!Player->Open(Url, Options, PlayerOptions)) { NotifyLifetimeManagerDelegate_PlayerCreateFailed(); CurrentUrl.Empty(); ActivePlayerOptions.Reset(); return false; } { FScopeLock Lock(&LastTimeValuesCS); BlockOnRangeDisabled = false; BlockOnRange.OnFlush(); LastVideoSampleProcessedTimeRange = TRange::Empty(); LastAudioSampleProcessedTime.Invalidate(); CurrentFrameVideoTimeStamp.Invalidate(); CurrentFrameVideoDisplayTimeStamp.Invalidate(); CurrentFrameAudioTimeStamp.Invalidate(); NextEstVideoTimeAtFrameStart.Invalidate(); SeekTargetTime.Invalidate(); NextSeekTime.Reset(); NextSequenceIndex.Reset(); if (Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { NextSequenceIndex = 0; } } ResetTracks(); if (bCreateNewPlayer) { NotifyLifetimeManagerDelegate_PlayerCreated(); } return true; } void FMediaPlayerFacade::QueryCacheState(EMediaTrackType TrackType, EMediaCacheState State, TRangeSet& OutTimeRanges) const { if (!Player.IsValid()) { return; } if (State == EMediaCacheState::Cached) { if (TrackType == EMediaTrackType::Audio) { Cache->GetCachedAudioSampleRanges(OutTimeRanges); } else if (TrackType == EMediaTrackType::Video) { Cache->GetCachedVideoSampleRanges(OutTimeRanges); } } else { if (TrackType == EMediaTrackType::Video) { Player->GetCache().QueryCacheState(State, OutTimeRanges); } } } bool FMediaPlayerFacade::Seek(const FTimespan& InTime) { NextSeekTime.Reset(); auto CurrentPlayer = Player; if (!CurrentPlayer.IsValid()) { return false; } FTimespan Duration = CurrentPlayer->GetControls().GetDuration(); FTimespan Time; if (IsDurationValidAndFinite(Duration)) { const TRange ActiveRange = GetActivePlaybackRange(); if (CurrentPlayer->GetControls().IsLooping()) { const FTimespan ActiveRangeDuration = ActiveRange.GetUpperBoundValue() - ActiveRange.GetLowerBoundValue(); Time = WrappedModulo(InTime - ActiveRange.GetLowerBoundValue(), ActiveRangeDuration) + ActiveRange.GetLowerBoundValue(); } else { Time = FTimespan(FMath::Clamp(InTime.GetTicks(), ActiveRange.GetLowerBoundValue().GetTicks(), ActiveRange.GetUpperBoundValue().GetTicks())); } } else { Time = InTime; } FMediaSeekParams SeekParams; // v2 timing players are *required* to use the new sequence index we set up. TOptional SequenceIndexNow = NextSequenceIndex; if (CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { SeekParams.NewSequenceIndex = NextSequenceIndex = NextSequenceIndex.Get(0) + 1; } // Issue the seek. if (!CurrentPlayer->GetControls().Seek(Time, SeekParams)) { // If that failed restore the sequence index. NextSequenceIndex = SequenceIndexNow; return false; } FScopeLock Lock(&CriticalSection); // V2 timing player? if (CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Yes. Flush only the facade side of the system as needed for seeks // (the player is expected to flush its internal queues as needed itself) PrepareSampleQueueForSequenceIndex(); Flush(CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::PlayerUsesInternalFlushOnSeek), true); } else { // No. Flush as requested... if (CurrentPlayer->FlushOnSeekStarted()) { Flush(CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::PlayerUsesInternalFlushOnSeek), false); } } SeekTargetTime = FMediaTimeStamp(Time, NextSequenceIndex.Get(0), 0); return true; } bool FMediaPlayerFacade::IsSeeking() const { // Code inspection notes: // - Usually protected by LastTimeValuesCS, but sometimes CriticalSection (or both interlocked). // - GetCurrentPlaybackTimeRange reads it outside of a scope lock. FScopeLock Lock(&LastTimeValuesCS); return SeekTargetTime.IsValid(); } FMediaTimeStamp FMediaPlayerFacade::GetSeekTarget() const { FScopeLock Lock(&LastTimeValuesCS); return SeekTargetTime; } void FMediaPlayerFacade::SetNextSeek(const FTimespan& InTime) { NextSeekTime = InTime; } void FMediaPlayerFacade::SetBlockOnTime(const FTimespan& Time) { #if !MEDIAPLAYERFACADE_DISABLE_BLOCKING if (!Player.IsValid() || !Player->GetControls().CanControl(EMediaControl::BlockOnFetch)) { return; } if (Time == FTimespan::MinValue()) { BlockOnRange.SetRange(TRange::Empty()); Player->GetControls().SetBlockingPlaybackHint(false); } else { TRange Range = TRange::Inclusive(Time, Time); BlockOnRange.SetRange(Range); Player->GetControls().SetBlockingPlaybackHint(true); } #endif } void FMediaPlayerFacade::SetBlockOnTimeRange(const TRange& TimeRange) { #if !MEDIAPLAYERFACADE_DISABLE_BLOCKING BlockOnRange.SetRange(TimeRange); #endif } void FMediaPlayerFacade::FBlockOnRange::OnFlush() { LastProcessedTimeRange = TRange::Empty(); OnBlockSequenceIndex = 0; OnBlockLoopIndexOffset = 0; RangeIsDirty = true; } void FMediaPlayerFacade::FBlockOnRange::OnSeek(int32 PrimaryIndex) { LastProcessedTimeRange = TRange::Empty(); OnBlockSequenceIndex = PrimaryIndex; OnBlockLoopIndexOffset = 0; RangeIsDirty = true; } void FMediaPlayerFacade::FBlockOnRange::SetRange(const TRange& NewRange) { if (CurrentTimeRange != NewRange) { CurrentTimeRange = NewRange; RangeIsDirty = true; } } bool FMediaPlayerFacade::FBlockOnRange::IsSet() const { TSharedPtr CurrentPlayer(Facade->Player); check(CurrentPlayer.IsValid()); if (!RangeIsDirty) { return !BlockOnRange.IsEmpty(); } return (!CurrentTimeRange.IsEmpty() && CurrentPlayer->GetControls().CanControl(EMediaControl::BlockOnFetch)); } const TRange& FMediaPlayerFacade::FBlockOnRange::GetRange() const { TSharedPtr CurrentPlayer(Facade->Player); check(CurrentPlayer.IsValid()); if (!RangeIsDirty) { return BlockOnRange; } // If the range is empty or the player can't support blocked playback: reset everything & return empty block range... if (CurrentTimeRange.IsEmpty() || !CurrentPlayer->GetControls().CanControl(EMediaControl::BlockOnFetch)) { LastProcessedTimeRange = TRange::Empty(); BlockOnRange = TRange::Empty(); CurrentPlayer->GetControls().SetBlockingPlaybackHint(false); return BlockOnRange; } EMediaState PlayerState = CurrentPlayer->GetControls().GetState(); if (PlayerState != EMediaState::Paused && PlayerState != EMediaState::Playing) { // Return an empty range. Note that the "IsSet()" method will still report a set block - so all code will remain in "external clock" mode, // but no samples will be requested (and no actual blocking should take place) static auto EmptyRange(TRange::Empty()); return EmptyRange; } const int64 StartTicks = CurrentTimeRange.GetLowerBoundValue().GetTicks(); const int64 EndExclusiveTicks = CurrentTimeRange.GetUpperBoundValue().GetTicks(); const int64 DurationTicks = CurrentPlayer->GetControls().GetDuration().GetTicks(); check(StartTicks >= 0 && EndExclusiveTicks >= StartTicks); /* When looping we need to synthesize the expected start and end loop indices of the range we are returning. This is because on playback start/seeking the media player implicitly starts with a loop index of 0, while we could be in any of the looping repetitions in the sequencer track of this movie (if the track has been pulled out "to the right" to have it repeat the clip n times). Because of that we need to "lock" an initial loop offset that represents this initial difference and needs to be adjusted with any looping of the sequencer. Please note that this does not include the case where the media track ends before the end of the sequencer. In that case the media player is closed and re-opened on the media track boundaries. */ if (!CurrentPlayer->GetControls().IsLooping()) { FTimespan Start(CurrentTimeRange.GetLowerBoundValue()); FTimespan End(CurrentTimeRange.GetUpperBoundValue()); const int32 LoopIdx = LastProcessedTimeRange.IsEmpty() ? 0 : LastProcessedTimeRange.GetLowerBoundValue().GetTicks() / DurationTicks; BlockOnRange = TRange(FMediaTimeStamp(Start, OnBlockSequenceIndex, OnBlockLoopIndexOffset + LoopIdx), FMediaTimeStamp(End, OnBlockSequenceIndex, OnBlockLoopIndexOffset + LoopIdx)); } else { /* If this would be called very early in the player's startup after open() we would not yet be known... that would be fatal Should this actually happen in real-life applications, we could move the computations here into an accessor method used internally, so that this would be done only if data is processed, which would also mean: we know the duration! (Exception: live playback! --> but we would not allow blocking there anyway! (makes no sense as real life use case)) */ if (!IsDurationValidAndFinite(CurrentPlayer->GetControls().GetDuration())) { // Catch if this is called to early and reset blocking... BlockOnRange = TRange::Empty(); CurrentPlayer->GetControls().SetBlockingPlaybackHint(false); return BlockOnRange; } FMediaTimeStamp t0, t1; t0.SetTime(FTimespan(StartTicks % DurationTicks)).SetSequenceIndex(OnBlockSequenceIndex).SetLoopIndex(StartTicks / DurationTicks); t1.SetTime(FTimespan(EndExclusiveTicks% DurationTicks)).SetSequenceIndex(OnBlockSequenceIndex).SetLoopIndex(EndExclusiveTicks / DurationTicks); const bool bReverse = (Facade->GetUnpausedRate() < 0.0f); // Did we process a time range before? if (LastProcessedTimeRange.IsEmpty()) { /* No, playback has just started fresh or through a seek. The media player will start with a loop index of 0, but the blocking range could be anywhere within a movie repetition (ie when the movie has been pulled out in the sequencer track to repeat a number of times). We set that repetion count as the base for the loop index. */ check(OnBlockLoopIndexOffset == 0); OnBlockLoopIndexOffset = !bReverse ? -t0.GetLoopIndex() : -t1.GetLoopIndex(); } else { /* We already processed a time range. We now need to check if the current one has wrapped around in the current playback direction. Theoretically, with either very, very short movies or an excessively huge delta time this could have wrapped around more than once. We cannot detect this and hope this will not occur. */ if (!bReverse) { if (LastProcessedTimeRange.GetLowerBoundValue() > CurrentTimeRange.GetLowerBoundValue()) { // Figure the loop index of the last range's start time. const int32 LastRangeLoopIdx = LastProcessedTimeRange.GetLowerBoundValue().GetTicks() / DurationTicks; OnBlockLoopIndexOffset += LastRangeLoopIdx + 1; } } else { if (LastProcessedTimeRange.GetLowerBoundValue() < CurrentTimeRange.GetLowerBoundValue()) { // Figure the loop index of the last range's start time. const int32 ThisRangeLoopIdx = t0.GetLoopIndex(); OnBlockLoopIndexOffset -= ThisRangeLoopIdx + 1; } } } // Assemble final blocking range t0.AdjustLoopIndex(OnBlockLoopIndexOffset); t1.AdjustLoopIndex(OnBlockLoopIndexOffset); BlockOnRange = TRange(t0, t1); check(!BlockOnRange.IsEmpty()); } // Note: Due to varying DTs the new range will NOT be a simple monotone progression in playback direction, but might overlap or even be a subset of the previous one // We do not put any safeguards in place here, but rather use the "is last sample still valid" logic to reject illogical / impossible range requests. // All that aside: we DO expect ranges start (lower bound if forward, upper if reverse playback) to be moving in a monotone manner according to the set playback direction. CurrentPlayer->GetControls().SetBlockingPlaybackHint(!BlockOnRange.IsEmpty()); LastProcessedTimeRange = CurrentTimeRange; RangeIsDirty = false; return BlockOnRange; } void FMediaPlayerFacade::SetCacheWindow(FTimespan Ahead, FTimespan Behind) { Cache->SetCacheWindow(Ahead, Behind); } void FMediaPlayerFacade::SetGuid(FGuid& Guid) { PlayerGuid = Guid; } bool FMediaPlayerFacade::SetLooping(bool Looping) { return Player.IsValid() && Player->GetControls().SetLooping(Looping); } void FMediaPlayerFacade::SetMediaOptions(const IMediaOptions* Options) { } bool FMediaPlayerFacade::SetRate(float Rate) { // Enter CS as we change the rate which we read on the tickable thread FScopeLock Lock(&CriticalSection); if (!Player.IsValid()) { return false; } // Is this new rate supported? bool bRateOk = true; if (Rate != 0.0f && !(Player->GetControls().GetSupportedRates(EMediaRateThinning::Thinned).Contains(Rate) || Player->GetControls().GetSupportedRates(EMediaRateThinning::Unthinned).Contains(Rate))) { // Pause player instead... // (some players may do this as a reaction to the illegal rate anyways - but we need to track the state properly!) Rate = 0.0f; bRateOk = false; } // Attempt to set the rate... if (!Player->GetControls().SetRate(Rate)) { return false; } // Any change? if (CurrentRate == Rate) { // no change - just return with ok status return bRateOk; } // Notify sinks of rate change FMediaSampleSinkEventData Data; Data.PlaybackRateChanged.PlaybackRate = Rate; SendSinkEvent(EMediaSampleSinkEvent::PlaybackRateChanged, Data); if ((LastRate * Rate) < 0.0f) { // direction change Flush(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2), false); } else { if (Rate == 0.0f) { // Invalidate audio time on entering pause mode... if (TSharedPtr AudioSink = PrimaryAudioSink.Pin()) { AudioSink->InvalidateAudioTime(); } } } // Track last "unpaused" rate we set if (Rate != 0.0) { LastRate = Rate; } CurrentRate = Rate; return bRateOk; } bool FMediaPlayerFacade::SetNativeVolume(float Volume) { return Player.IsValid() ? Player->SetNativeVolume(Volume) : false; } bool FMediaPlayerFacade::SetTrackFormat(EMediaTrackType TrackType, int32 TrackIndex, int32 FormatIndex) { return Player.IsValid() ? Player->GetTracks().SetTrackFormat((EMediaTrackType)TrackType, TrackIndex, FormatIndex) : false; } bool FMediaPlayerFacade::SetVideoTrackFrameRate(int32 TrackIndex, int32 FormatIndex, float FrameRate) { return Player.IsValid() ? Player->GetTracks().SetVideoTrackFrameRate(TrackIndex, FormatIndex, FrameRate) : false; } bool FMediaPlayerFacade::SetViewField(float Horizontal, float Vertical, bool Absolute) { return Player.IsValid() && Player->GetView().SetViewField(Horizontal, Vertical, Absolute); } bool FMediaPlayerFacade::SetViewOrientation(const FQuat& Orientation, bool Absolute) { return Player.IsValid() && Player->GetView().SetViewOrientation(Orientation, Absolute); } bool FMediaPlayerFacade::SupportsRate(float Rate, bool Unthinned) const { EMediaRateThinning Thinning = Unthinned ? EMediaRateThinning::Unthinned : EMediaRateThinning::Thinned; return Player.IsValid() && Player->GetControls().GetSupportedRates(Thinning).Contains(Rate); } TRange FMediaPlayerFacade::GetPlaybackTimeRange(EMediaTimeRangeType InRangeToGet) const { TSharedPtr CurrentPlayer(Player); return CurrentPlayer.IsValid() ? CurrentPlayer->GetControls().GetPlaybackTimeRange(InRangeToGet) : TRange(); } bool FMediaPlayerFacade::SetPlaybackTimeRange(const TRange& InTimeRange) { TSharedPtr CurrentPlayer(Player); return CurrentPlayer.IsValid() ? CurrentPlayer->GetControls().SetPlaybackTimeRange(InTimeRange) : false; } void FMediaPlayerFacade::SetLastAudioRenderedSampleTime(FTimespan SampleTime) { FScopeLock Lock(&LastTimeValuesCS); LastAudioRenderedSampleTime.TimeStamp = FMediaTimeStamp(SampleTime); LastAudioRenderedSampleTime.SampledAtTime = FPlatformTime::Seconds(); } FTimespan FMediaPlayerFacade::GetLastAudioRenderedSampleTime() const { FScopeLock Lock(&LastTimeValuesCS); return LastAudioRenderedSampleTime.TimeStamp.Time; } void FMediaPlayerFacade::SetAreEventsSafeForAnyThread(bool bInAreEventsSafeForAnyThread) { bAreEventsSafeForAnyThread = bInAreEventsSafeForAnyThread; } /* FMediaPlayerFacade implementation *****************************************************************************/ bool FMediaPlayerFacade::BlockOnFetch() const { check(Player.IsValid()); const TRange BR(GetAdjustedBlockOnRange()); if (BR.IsEmpty() || !Player->GetControls().CanControl(EMediaControl::BlockOnFetch) || BlockOnRangeDisabled || bHaveActiveAudio) { return false; // no blocking requested / not supported / audio present } if (Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // // V2 blocking logic // // note: with V2 timing we only get here if any current sample is no longer considered "valid" and we didn't so far get a new one that would be // --> we do not need to check the actual range here; we only check for exceptions, where we can proceed although we don't have the sample... // The next checks make only sense if the player is done preparing... if (!IsPreparing()) { // Looping off? if (!Player->GetControls().IsLooping()) { // Yes. Is the sample outside the media's range? // (note: this assumes the media starts at time ZERO - this will not be the case at all times (e.g. live playback) -- for now we assume a player will flagged blocked playback as invalid in that case!) if (BR.GetUpperBoundValue().GetTime() < FTimespan::Zero() || Player->GetControls().GetDuration() <= BR.GetLowerBoundValue().GetTime()) { return false; } } } // Block until sample arrives! return true; } else { // // V1 blocking logic // if (IsPreparing()) { return true; // block on media opening } if (!IsPlaying()) { // no blocking if we are not playing (e.g. paused) return false; } if (CurrentRate < 0.0f) { return false; // block only in forward play } const bool VideoReady = (VideoSampleSinks.Num() == 0) || (BR.GetUpperBoundValue().Time < NextVideoSampleTime); if (VideoReady) { return false; // video is ready } return true; } } void FMediaPlayerFacade::Flush(bool bExcludePlayer, bool bOnSeek) { UE_LOG(LogMediaUtils, Verbose, TEXT("PlayerFacade: Flushing sinks")); FScopeLock Lock(&CriticalSection); auto RawMediaPlayer = MediaPlayer.Get(); AudioSampleSinks.Flush(RawMediaPlayer); CaptionSampleSinks.Flush(RawMediaPlayer); MetadataSampleSinks.Flush(RawMediaPlayer); SubtitleSampleSinks.Flush(RawMediaPlayer); VideoSampleSinks.Flush(RawMediaPlayer); MostRecentlyDeliveredVideoFrameTimecode.Reset(); if (Player.IsValid() && !bExcludePlayer) { Player->GetSamples().FlushSamples(); } LastAudioRenderedSampleTime.Invalidate(); if (bOnSeek) { BlockOnRange.OnSeek(NextSequenceIndex.Get(0)); } else { BlockOnRange.OnFlush(); } if (Player.IsValid() && Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Logically we have no old sample anymore if we did seek // (as in: we will start asking for a new one until we get one - even with a rate of zero, if we had a non-zero one ever before) if (bOnSeek) { LastVideoSampleProcessedTimeRange = TRange::Empty(); } else { if (!bExcludePlayer && !LastVideoSampleProcessedTimeRange.IsEmpty()) { // Players will reset their sequence index related values, but keep the playback position. Adjust our record accordingly... int32 LoopIdxS = LastVideoSampleProcessedTimeRange.GetLowerBoundValue().GetLoopIndex(); int32 LoopIdxE = LastVideoSampleProcessedTimeRange.GetUpperBoundValue().GetLoopIndex(); LastVideoSampleProcessedTimeRange.SetLowerBoundValue(FMediaTimeStamp(LastVideoSampleProcessedTimeRange.GetLowerBoundValue().Time, 0, 0)); LastVideoSampleProcessedTimeRange.SetUpperBoundValue(FMediaTimeStamp(LastVideoSampleProcessedTimeRange.GetUpperBoundValue().Time, 0, LoopIdxE - LoopIdxS)); } } // Invalidate next video time to fetch (non-audio case) NextEstVideoTimeAtFrameStart.Invalidate(); // ...and seek target SeekTargetTime.Invalidate(); } NextVideoSampleTime = FTimespan::MinValue(); } void FMediaPlayerFacade::SendSinkEvent(EMediaSampleSinkEvent Event, const FMediaSampleSinkEventData& Data) { { FScopeLock Lock(&CriticalSection); AudioSampleSinks.ReceiveEvent(Event, Data); MetadataSampleSinks.ReceiveEvent(Event, Data); } CaptionSampleSinks.ReceiveEvent(Event, Data); SubtitleSampleSinks.ReceiveEvent(Event, Data); VideoSampleSinks.ReceiveEvent(Event, Data); } bool FMediaPlayerFacade::GetAudioTrackFormat(int32 TrackIndex, int32 FormatIndex, FMediaAudioTrackFormat& OutFormat) const { if (TrackIndex == INDEX_NONE) { TrackIndex = GetSelectedTrack(EMediaTrackType::Audio); } if (FormatIndex == INDEX_NONE) { FormatIndex = GetTrackFormat(EMediaTrackType::Audio, TrackIndex); } return (Player.IsValid() && Player->GetTracks().GetAudioTrackFormat(TrackIndex, FormatIndex, OutFormat)); } IMediaPlayerFactory* FMediaPlayerFacade::GetPlayerFactoryForUrl(const FString& Url, const IMediaOptions* Options) const { FName PlayerName; if (DesiredPlayerName != NAME_None) { PlayerName = DesiredPlayerName; } else if (Options != nullptr) { PlayerName = Options->GetDesiredPlayerName(); } else { PlayerName = NAME_None; } if (MediaModule == nullptr) { UE_LOG(LogMediaUtils, Error, TEXT("Failed to load Media module")); return nullptr; } // // Reuse existing player if explicitly requested name matches // if (Player.IsValid()) { IMediaPlayerFactory* CurrentFactory = MediaModule->GetPlayerFactory(Player->GetPlayerPluginGUID()); if (PlayerName == CurrentFactory->GetPlayerName()) { return CurrentFactory; } } // // Try to create explicitly requested player // if (PlayerName != NAME_None) { IMediaPlayerFactory* Factory = MediaModule->GetPlayerFactory(PlayerName); if (Factory == nullptr) { UE_LOG(LogMediaUtils, Error, TEXT("Could not find desired player %s for %s"), *PlayerName.ToString(), *Url); } return Factory; } // // Try to find a fitting player with no explicit name given // const TArray& PlayerFactories = MediaModule->GetPlayerFactories(); if (PlayerFactories.Num() == 0) { UE_LOG(LogMediaUtils, Error, TEXT("Cannot play %s: no media player plug-ins are installed and enabled in this project"), *Url); return nullptr; } struct FCandidate { FName Name; IMediaPlayerFactory* Factory = nullptr; int32 ConfidenceScore = 0; }; TArray Candidates; const FString RunningPlatformName(FPlatformProperties::IniPlatformName()); for(IMediaPlayerFactory* Factory : PlayerFactories) { if (Factory->SupportsPlatform(RunningPlatformName)) { int32 ConfidenceScore = Factory->GetPlayabilityConfidenceScore(Url, Options, nullptr, nullptr); if (ConfidenceScore > 0) { FCandidate& Candidate = Candidates.Emplace_GetRef(); Candidate.Name = Factory->GetPlayerName(); Candidate.Factory = Factory; Candidate.ConfidenceScore = ConfidenceScore; } } } Candidates.Sort([](const FCandidate& c1, const FCandidate& c2) { // If both factories are equally confident, sort alphabetically by name. if (c1.ConfidenceScore == c2.ConfidenceScore) { return c1.Name.ToString() < c2.Name.ToString(); } // Sort by descending confidence score. return c1.ConfidenceScore > c2.ConfidenceScore; }); if (Candidates.Num()) { return Candidates[0].Factory; } // // No suitable player found! // UE_LOG(LogMediaUtils, Error, TEXT("Cannot play %s, because none of the enabled media player plug-ins support it:"), *Url); for (IMediaPlayerFactory* Factory : PlayerFactories) { if (Factory->SupportsPlatform(RunningPlatformName)) { UE_LOG(LogMediaUtils, Log, TEXT("| %s (URI scheme or file extension not supported)"), *Factory->GetPlayerName().ToString()); } else { UE_LOG(LogMediaUtils, Log, TEXT("| %s (only available on %s, but not on %s)"), *Factory->GetPlayerName().ToString(), *FString::Join(Factory->GetSupportedPlatforms(), TEXT(", ")), *RunningPlatformName); } } return nullptr; } bool FMediaPlayerFacade::GetVideoTrackFormat(int32 TrackIndex, int32 FormatIndex, FMediaVideoTrackFormat& OutFormat) const { if (TrackIndex == INDEX_NONE) { TrackIndex = GetSelectedTrack(EMediaTrackType::Video); } if (FormatIndex == INDEX_NONE) { FormatIndex = GetTrackFormat(EMediaTrackType::Video, TrackIndex); } return (Player.IsValid() && Player->GetTracks().GetVideoTrackFormat(TrackIndex, FormatIndex, OutFormat)); } void FMediaPlayerFacade::ProcessEvent(EMediaEvent Event, bool bIsBroadcastAllowed) { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeProcessEvent); if ((Event == EMediaEvent::MediaOpened) || (Event == EMediaEvent::MediaOpenFailed)) { if (Event == EMediaEvent::MediaOpenFailed) { CurrentUrl.Empty(); } const FString MediaInfo = Player.IsValid() ? Player->GetInfo() : TEXT(""); if (MediaInfo.IsEmpty()) { UE_LOG(LogMediaUtils, Verbose, TEXT("PlayerFacade: Media Info: n/a")); } else { UE_LOG(LogMediaUtils, Verbose, TEXT("PlayerFacade: Media Info:\n%s"), *MediaInfo); } } else if (Event == EMediaEvent::TracksChanged) { SelectDefaultTracks(); if (Player.IsValid()) { // Apply track selection immediately so the selection can be queried. UpdateTrackSelectionWithPlayer(); if (!Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Execute flush for older players only Flush(); } } } else if (Event == EMediaEvent::SeekCompleted) { // We only consider flushing on seek completion if there is a V1 timing player... if (Player.IsValid() && !Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Does the player want this? if (Player->FlushOnSeekCompleted()) { Flush(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::PlayerUsesInternalFlushOnSeek), true); } } } else if (Event == EMediaEvent::MediaClosed) { // Player still closed? if (CurrentUrl.IsEmpty()) { // Yes, this also means: if we still have a player, it's still the one this event originated from FMediaSampleSinkEventData Data; SendSinkEvent(EMediaSampleSinkEvent::MediaClosed, Data); // If player allows: close it down all the way right now if (Player.IsValid() && Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::AllowShutdownOnClose)) { bDidRecentPlayerHaveError = HasError(); DestroyPlayer(); } // Stop issuing audio thread ticks until we open the player again MediaModule->GetTicker().RemoveTickable(AsShared()); } } else if (Event == EMediaEvent::PlaybackEndReached) { if (Player.IsValid() && !Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // Execute flush for older players only Flush(); } FMediaSampleSinkEventData Data; SendSinkEvent(EMediaSampleSinkEvent::PlaybackEndReached, Data); } if (bIsBroadcastAllowed) { MediaEvent.Broadcast(Event); } else { QueuedEventBroadcasts.Enqueue(Event); } } void FMediaPlayerFacade::ResetTracks() { for (int32 Idx = 0; Idx < (int32)EMediaTrackType::Num; ++Idx) { TrackSelection.UserSelection[Idx] = -1; TrackSelection.PlayerSelection[Idx] = -1; } } void FMediaPlayerFacade::SelectDefaultTracks() { // See if the player has selected appropriate default tracks. TSharedPtr CurrentPlayer(Player); if (CurrentPlayer.IsValid() && CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::PlayerSelectsDefaultTracks)) { ResetTracks(); // Get what the player has selected as user defaults. // The TrackSelection.PlayerSelection[...] will be updated in UpdateTrackSelectionWithPlayer() // where the existence of sinks is checked for. IMediaTracks& Tracks = CurrentPlayer->GetTracks(); for(int32 Idx=0; Idx<(int32)EMediaTrackType::Num; ++Idx) { TrackSelection.UserSelection[Idx] = Tracks.GetSelectedTrack((EMediaTrackType)Idx); } // If overrides are set, use them. if (ActivePlayerOptions.IsSet()) { if (ActivePlayerOptions.GetValue().TrackSelection == EMediaPlayerOptionTrackSelectMode::UseTrackOptionIndices) { FMediaPlayerTrackOptions TrackOptions; TrackOptions = ActivePlayerOptions.GetValue().Tracks; TrackSelection.UserSelection[(int32)EMediaTrackType::Audio] = TrackOptions.Audio; TrackSelection.UserSelection[(int32)EMediaTrackType::Caption] = TrackOptions.Caption; TrackSelection.UserSelection[(int32)EMediaTrackType::Metadata] = TrackOptions.Metadata; TrackSelection.UserSelection[(int32)EMediaTrackType::Subtitle] = TrackOptions.Subtitle; TrackSelection.UserSelection[(int32)EMediaTrackType::Video] = TrackOptions.Video; } } } else { FMediaPlayerTrackOptions TrackOptions; if (ActivePlayerOptions.IsSet()) { if (ActivePlayerOptions.GetValue().TrackSelection == EMediaPlayerOptionTrackSelectMode::UseTrackOptionIndices) { TrackOptions = ActivePlayerOptions.GetValue().Tracks; } } TrackSelection.UserSelection[(int32)EMediaTrackType::Audio] = TrackOptions.Audio; TrackSelection.UserSelection[(int32)EMediaTrackType::Caption] = TrackOptions.Caption; TrackSelection.UserSelection[(int32)EMediaTrackType::Metadata] = TrackOptions.Metadata; TrackSelection.UserSelection[(int32)EMediaTrackType::Subtitle] = TrackOptions.Subtitle; TrackSelection.UserSelection[(int32)EMediaTrackType::Video] = TrackOptions.Video; } } bool FMediaPlayerFacade::SelectTrack(EMediaTrackType TrackType, int32 TrackIndex) { TSharedPtr CurrentPlayer(Player); if (CurrentPlayer.IsValid()) { IMediaTracks& Tracks = CurrentPlayer->GetTracks(); if (Tracks.GetNumTracks(TrackType) > TrackIndex) { TrackSelection.UserSelection[(int32)TrackType] = TrackIndex; return true; } } return false; } int32 FMediaPlayerFacade::GetSelectedTrack(EMediaTrackType TrackType) const { return TrackSelection.UserSelection[(int32)TrackType]; } void FMediaPlayerFacade::UpdateTrackSelectionWithPlayer() { check(Player.IsValid()); bool bChanges = false; IMediaTracks& Tracks = Player->GetTracks(); for (int32 Idx = 0; Idx < (int32)EMediaTrackType::Num; ++Idx) { // Player and user selection are different? if (TrackSelection.PlayerSelection[Idx] != TrackSelection.UserSelection[Idx]) { // Yes... int32 UserSelection = TrackSelection.UserSelection[Idx]; // Filter selection against the configured sinks... if (UserSelection != -1) { if ((Idx == (int)EMediaTrackType::Audio && !PrimaryAudioSink.IsValid()) || (Idx == (int)EMediaTrackType::Video && VideoSampleSinks.IsEmpty()) || (Idx == (int)EMediaTrackType::Caption && CaptionSampleSinks.IsEmpty()) || (Idx == (int)EMediaTrackType::Subtitle && SubtitleSampleSinks.IsEmpty()) || (Idx == (int)EMediaTrackType::Metadata && MetadataSampleSinks.IsEmpty())) { UserSelection = -1; } } // After filtering the user's selection, do we still have to change things? if (TrackSelection.PlayerSelection[Idx] != UserSelection) { // Yes! if (Tracks.SelectTrack((EMediaTrackType)Idx, UserSelection)) { // Recall what is now selected with the player... TrackSelection.PlayerSelection[Idx] = UserSelection; bChanges = true; } else { // Track selection failed. Patch the user selection to be what we know of the player's, so we do not reattempt this over and over... TrackSelection.UserSelection[Idx] = TrackSelection.PlayerSelection[Idx]; } } } } if (bChanges) { if (!Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::IsTrackSwitchSeamless)) { Flush(); } } } float FMediaPlayerFacade::GetUnpausedRate() const { return (CurrentRate == 0.0f) ? LastRate : CurrentRate; } /* IMediaClockSink interface *****************************************************************************/ void FMediaPlayerFacade::TickInput(FTimespan DeltaTime, FTimespan Timecode) { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeTickInput); if (Player.IsValid()) { UpdateTrackSelectionWithPlayer(); MonitorAudioEnablement(); Player->TickInput(DeltaTime, Timecode); bool bIsBroadcastAllowed = bAreEventsSafeForAnyThread || IsInGameThread(); if (Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // // New timing control (handled before any engine world, object etc. updates; so "all frame" (almost) see the state produced here) // // process deferred events // NOTE: if there is no player anymore we execute the remaining queued events in TickFetch (backwards compatibility - should move here once V1 support removed) EMediaEvent Event; if (bIsBroadcastAllowed) { while(QueuedEventBroadcasts.Dequeue(Event)) { MediaEvent.Broadcast(Event); } } while(QueuedEvents.Dequeue(Event)) { ProcessEvent(Event, bIsBroadcastAllowed); } // Handling events may have killed the player. Did it? if (!Player.IsValid()) { // If so: nothing more to do! return; } // // Setup timing for sample processing // PreSampleProcessingTimeHandling(); TRange TimeRange; if (!GetCurrentPlaybackTimeRange(TimeRange, CurrentRate, DeltaTime, false)) { return; } SET_FLOAT_STAT(STAT_MediaUtils_FacadeTime, TimeRange.GetLowerBoundValue().Time.GetTotalSeconds()); // // Process samples in range // IMediaSamples& Samples = Player->GetSamples(); double BlockingStart = FPlatformTime::Seconds(); while(1) { ProcessCaptionSamples(Samples, TimeRange); ProcessSubtitleSamples(Samples, TimeRange); if (ProcessVideoSamples(Samples, TimeRange)) { // We either got a new sample or a current one is still the best choice... break; } // The current one is outdated and no new one was delivered. Should we block for one? if (!BlockOnFetch()) { // No... continue... break; } // Issue tick call with dummy timing as some players advance some state in the tick, which we wait for Player->TickInput(FTimespan::Zero(), FTimespan::MinValue()); // Monitor / update seek status UpdateSeekStatus(); // Process deferred events & check for events that break the block bool bEventCancelsBlock = false; while(QueuedEvents.Dequeue(Event)) { if (Event == EMediaEvent::MediaClosed || Event == EMediaEvent::MediaOpenFailed) { bEventCancelsBlock = true; } ProcessEvent(Event, bIsBroadcastAllowed); } // We might have lost the player during event handling or an event breaks the block... if (!Player.IsValid() || bEventCancelsBlock) { // Disable blocking feature for now (a new open would reset this) UE_LOG(LogMediaUtils, Warning, TEXT("Blocking media playback closed or failed. Disabling it for this playback session.")); BlockOnRangeDisabled = true; break; } // Timeout? if ((FPlatformTime::Seconds() - BlockingStart) > static_cast(UE::MediaUtils::Private::GBlockOnFetchTimeout)) { FString Url; #if !UE_BUILD_SHIPPING Url = Player->GetUrl(); #endif // !UE_BUILD_SHIPPING UE_LOG(LogMediaUtils, Error, TEXT("Blocking media playback timed out. Disabling it for this playback session. URL:%s"), *Url); BlockOnRangeDisabled = true; break; } FPlatformProcess::Sleep(0.0f); } SET_DWORD_STAT(STAT_MediaUtils_FacadeNumVideoSamples, Samples.NumVideoSamples()); // // Advance timing etc. // PostSampleProcessingTimeHandling(DeltaTime); if (bHaveActiveAudio) { // Keep currently last processed audio sample timestamp available for all frame (to provide consistent info) FScopeLock Lock(&LastTimeValuesCS); CurrentFrameAudioTimeStamp = LastAudioSampleProcessedTime.TimeStamp; } } // Check if primary audio sink needs a change and make sure invalid sinks are purged at all times PrimaryAudioSink = AudioSampleSinks.GetPrimaryAudioSink(); } } void FMediaPlayerFacade::TickFetch(FTimespan DeltaTime, FTimespan Timecode) { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeTickFetch); TSharedPtr CurrentPlayer(Player); if (!CurrentPlayer.IsValid()) { // Send out deferred broadcasts. EMediaEvent Event; bool bIsBroadcastAllowed = bAreEventsSafeForAnyThread || IsInGameThread(); if (bIsBroadcastAllowed) { while(QueuedEventBroadcasts.Dequeue(Event)) { MediaEvent.Broadcast(Event); } } // process deferred events while(QueuedEvents.Dequeue(Event)) { ProcessEvent(Event, bIsBroadcastAllowed); } return; } if (!CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { // // Old timing control // // let the player generate samples & process events CurrentPlayer->TickFetch(DeltaTime, Timecode); { // process deferred events EMediaEvent Event; while(QueuedEvents.Dequeue(Event)) { ProcessEvent(Event, true); } } TRange TimeRange; const FTimespan CurrentTime = GetTime(); SET_FLOAT_STAT(STAT_MediaUtils_FacadeTime, CurrentTime.GetTotalSeconds()); // get current play rate float Rate = GetUnpausedRate(); if (Rate > 0.0f) { TimeRange = TRange::AtMost(CurrentTime); } else if (Rate < 0.0f) { TimeRange = TRange::AtLeast(CurrentTime); } else { TimeRange = TRange(CurrentTime); } // process samples in range IMediaSamples& Samples = CurrentPlayer->GetSamples(); const double BlockOnFetchTimeout = static_cast(UE::MediaUtils::Private::GBlockOnFetchTimeout); bool Blocked = false; FDateTime BlockedTime; while(true) { ProcessCaptionSamplesV1(Samples, TimeRange); ProcessSubtitleSamplesV1(Samples, TimeRange); ProcessVideoSamplesV1(Samples, TimeRange); if (!BlockOnFetch()) { break; } if (Blocked) { if ((FDateTime::UtcNow() - BlockedTime) >= FTimespan::FromSeconds(BlockOnFetchTimeout)) { UE_LOG(LogMediaUtils, Verbose, TEXT("PlayerFacade: Aborted block on fetch %s after %i seconds"), *BlockOnRange.GetRange().GetLowerBoundValue().Time.ToString(TEXT("%h:%m:%s.%t")), static_cast(BlockOnFetchTimeout) ); break; } } else { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Blocking on fetch %s"), *BlockOnRange.GetRange().GetLowerBoundValue().Time.ToString(TEXT("%h:%m:%s.%t"))); Blocked = true; BlockedTime = FDateTime::UtcNow(); } FPlatformProcess::Sleep(0.0f); } } } void FMediaPlayerFacade::TickOutput(FTimespan DeltaTime, FTimespan /*Timecode*/) { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeTickOutput); if (!Player.IsValid()) { return; } Cache->Tick(DeltaTime, CurrentRate, GetTime()); ExecuteNextSeek(); } /* IMediaTickable interface *****************************************************************************/ void FMediaPlayerFacade::TickTickable() { SCOPE_CYCLE_COUNTER(STAT_MediaUtils_FacadeTickTickable); FScopeLock Lock(&CriticalSection); if (!Player.IsValid()) { return; } float Rate = GetUnpausedRate(); if (Rate == 0.0f) { return; } { FScopeLock Lock1(&LastTimeValuesCS); Player->SetLastAudioRenderedSampleTime(LastAudioRenderedSampleTime.TimeStamp.Time); } Player->TickAudio(); // determine range of valid samples // process samples in range IMediaSamples& Samples = Player->GetSamples(); if (Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)) { ProcessAudioSamples(Samples, TRange()); const FMediaTimeStamp Time = GetTimeStamp(); auto TimeRange = TRange::Inclusive(FMediaTimeStamp(FTimespan::MinValue(), 0, 0), Time + MediaPlayerFacade::MetadataPreroll); ProcessMetadataSamples(Samples, TimeRange); } else { TRange AudioTimeRange; TRange MetadataTimeRange; const FTimespan Time = GetTime(); if (Rate >= 0.0f) { AudioTimeRange = TRange::Inclusive(FTimespan::MinValue(), Time + MediaPlayerFacade::AudioPreroll); MetadataTimeRange = TRange::Inclusive(FTimespan::MinValue(), Time + MediaPlayerFacade::MetadataPreroll); } else { AudioTimeRange = TRange::Inclusive(Time - MediaPlayerFacade::AudioPreroll, FTimespan::MaxValue()); MetadataTimeRange = TRange::Inclusive(Time - MediaPlayerFacade::MetadataPreroll, FTimespan::MaxValue()); } ProcessAudioSamplesV1(Samples, AudioTimeRange); ProcessMetadataSamplesV1(Samples, MetadataTimeRange); } SET_DWORD_STAT(STAT_MediaUtils_FacadeNumAudioSamples, Samples.NumAudioSamples()); } void FMediaPlayerFacade::PrepareSampleQueueForSequenceIndex() { FScopeLock Lock(&CriticalSection); if (!Player.IsValid() || !NextSequenceIndex.IsSet()) { return; } int32 MinSeqIdx = NextSequenceIndex.GetValue(); IMediaSamples& Samples = Player->GetSamples(); Samples.SetMinExpectedNextSequenceIndex(NextSequenceIndex); } void FMediaPlayerFacade::UpdateSeekStatus(const FMediaTimeStamp* pCheckTimeStamp) { check(Player.IsValid()); FScopeLock Lock(&CriticalSection); if (HaveVideoPlayback()) { if (SeekTargetTime.IsValid()) { // Either peek for the newest available sample or take a given timestamp to check against FMediaTimeStamp VideoTimeStamp; if (pCheckTimeStamp) { VideoTimeStamp = *pCheckTimeStamp; } else { Player->GetSamples().PeekVideoSampleTime(VideoTimeStamp); } if (VideoTimeStamp.IsValid() && VideoTimeStamp.GetSequenceIndex() >= NextSequenceIndex.Get(0)) { bool bRunningNonAudioClock = bHaveActiveAudio && !BlockOnRange.IsSet(); if (bRunningNonAudioClock) { NextEstVideoTimeAtFrameStart = FMediaTimeStampSample(VideoTimeStamp, FPlatformTime::Seconds()); } FScopeLock LockLT(&LastTimeValuesCS); CurrentFrameVideoDisplayTimeStamp = SeekTargetTime; SeekTargetTime.Invalidate(); } } } else { if (bHaveActiveAudio) { FScopeLock LockLT(&LastTimeValuesCS); if (CurrentFrameAudioTimeStamp >= SeekTargetTime) { SeekTargetTime.Invalidate(); } } else { // Neither audio nor video are presently active. We just assume we reached the seek target and continue... // (we currently have no other source of a current sample timestamp) SeekTargetTime.Invalidate(); } } } void FMediaPlayerFacade::ExecuteNextSeek() { if (NextSeekTime.IsSet() && !IsSeeking()) { if (!Seek(NextSeekTime.GetValue())) { // todo: signal error for failed seek. } } } void FMediaPlayerFacade::MonitorAudioEnablement() { // Update flag reflecting presence of audio in the current stream // (doing it just once per gameloop is enough) bool bHadActiveAudio = bHaveActiveAudio; bHaveActiveAudio = HaveAudioPlayback(); if (bHadActiveAudio && !bHaveActiveAudio) { // Reset state for dt-based playback so we grab a new PTS value immediately NextEstVideoTimeAtFrameStart.Invalidate(); } } void FMediaPlayerFacade::PreSampleProcessingTimeHandling() { check(Player.IsValid()); FScopeLock Lock(&CriticalSection); PrepareSampleQueueForSequenceIndex(); UpdateSeekStatus(); // No seeking? if (!SeekTargetTime.IsValid()) { // No seek pending & not paused. Can we / Do we need to prime a non-audio clock? if (!bHaveActiveAudio && !BlockOnRange.IsSet()) { // Nothing at all? if (!NextEstVideoTimeAtFrameStart.IsValid()) { // Try getting a new sample time to start things up... FMediaTimeStamp VideoTimeStamp; if (Player->GetSamples().PeekVideoSampleTime(VideoTimeStamp)) { NextEstVideoTimeAtFrameStart = FMediaTimeStampSample(VideoTimeStamp, FPlatformTime::Seconds()); } } else { // We have a time. But if we are actively playing forward... if (CurrentRate > 0.0f) { // ...and got some sample waiting for us... FMediaTimeStamp VideoTimeStamp; if (Player->GetSamples().PeekVideoSampleTime(VideoTimeStamp)) { // ...we need to see if the player's next sample might be so far in the future that we need to re-calibrate our timing // (this could happen if the stream has a "gap" in PTS values - e.g. after pausing a live feed from a camera) // (^^^ we do not do this on reverse playback as it is unlikely for such streams and might be thinned, hence show gaps under normal conditions) if (VideoTimeStamp.GetIndexValue() == NextEstVideoTimeAtFrameStart.TimeStamp.GetIndexValue()) { FTimespan Delta = VideoTimeStamp.Time - NextEstVideoTimeAtFrameStart.TimeStamp.Time; if (GetUnpausedRate() < 0.0f) { Delta = -Delta; } // Our threshold for re-calibration is twice the length of the last sample we got // (or 100ms if we have nothing) FTimespan DeltaLimit; if (!LastVideoSampleProcessedTimeRange.IsEmpty()) { DeltaLimit = LastVideoSampleProcessedTimeRange.Size().Time * 2; } else { DeltaLimit = FTimespan::FromSeconds(0.100); } if (Delta >= DeltaLimit) { NextEstVideoTimeAtFrameStart = FMediaTimeStampSample(VideoTimeStamp, FPlatformTime::Seconds()); } } } } } } } } void FMediaPlayerFacade::PostSampleProcessingTimeHandling(FTimespan DeltaTime) { check(Player.IsValid()); float Rate = CurrentRate; // No Audio clock? if (!bHaveActiveAudio) { // No external clock? (blocking) if (!BlockOnRange.IsSet()) { // Move video frame start estimate forward // (the initial NextEstVideoTimeAtFrameStart will never be valid if no video is present) if (NextEstVideoTimeAtFrameStart.IsValid()) { if (Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UseRealtimeWithVideoOnly)) { double NewBaseTime = FPlatformTime::Seconds(); NextEstVideoTimeAtFrameStart.TimeStamp.Time += FMath::TruncToInt64((NewBaseTime - NextEstVideoTimeAtFrameStart.SampledAtTime) * Rate); NextEstVideoTimeAtFrameStart.SampledAtTime = NewBaseTime; } else { NextEstVideoTimeAtFrameStart.TimeStamp.Time += DeltaTime * Rate; } // note: infinite duration (e.g. live playback - or players not yet supporting sequence indices on loops, when looping is enabled) // -> no need for special handling as FTimespan::MaxValue() is expected to be returned to signify this, which is quite "infinite" in practical terms FTimespan Duration = Player->GetControls().GetDuration(); const TRange ActiveRange = GetActivePlaybackRange(); const FTimespan ActiveRangeStart = ActiveRange.GetLowerBoundValue(); const FTimespan ActiveRangeEnd = ActiveRange.GetUpperBoundValue(); if (Player->GetControls().IsLooping()) { if (IsDurationValidAndFinite(Duration)) { const FTimespan ActiveRangeDuration = ActiveRange.GetUpperBoundValue() - ActiveRange.GetLowerBoundValue(); if (Rate >= 0.0f) { while(NextEstVideoTimeAtFrameStart.TimeStamp.Time >= ActiveRangeEnd) { NextEstVideoTimeAtFrameStart.TimeStamp.Time -= ActiveRangeDuration; NextEstVideoTimeAtFrameStart.TimeStamp.AdjustLoopIndex(1); } } else { while(NextEstVideoTimeAtFrameStart.TimeStamp.Time < ActiveRangeStart) { NextEstVideoTimeAtFrameStart.TimeStamp.Time += ActiveRangeDuration; NextEstVideoTimeAtFrameStart.TimeStamp.AdjustLoopIndex(-1); } } } } else { if (Rate >= 0.0f) { if (IsDurationValidAndFinite(Duration)) { if (NextEstVideoTimeAtFrameStart.TimeStamp.Time >= ActiveRangeEnd) { NextEstVideoTimeAtFrameStart.TimeStamp.Time = ActiveRangeEnd - FTimespan(1); } } } else { if (NextEstVideoTimeAtFrameStart.TimeStamp.Time < ActiveRangeStart) { NextEstVideoTimeAtFrameStart.TimeStamp.Time = ActiveRangeStart; } } } } } } } TRange FMediaPlayerFacade::GetActivePlaybackRange() const { TRange Rng(FTimespan::Zero(), FTimespan::Zero()); if (Player) { if (SupportsPlaybackTimeRange()) { Rng = GetPlaybackTimeRange(EMediaTimeRangeType::Current); } else { FTimespan Duration = Player->GetControls().GetDuration(); if (Duration <= FTimespan::Zero()) { Duration = FTimespan::MaxValue(); } Rng.SetUpperBound(Duration); } } return Rng; } bool FMediaPlayerFacade::GetCurrentPlaybackTimeRange(TRange& TimeRange, float Rate, FTimespan DeltaTime, bool bPurgeSampleRelated) const { /* * Note: while a seek operation is still in progress (no sample from target location has been processed) this will * return on an empty time range. */ check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); TSharedPtr AudioSink = PrimaryAudioSink.Pin(); if (bHaveActiveAudio && AudioSink.IsValid()) { // // Audio is available... // FMediaTimeStampSample AudioTime = AudioSink->GetAudioTime(); if (!AudioTime.IsValid()) { if (!bPurgeSampleRelated) { // If paused and not seeking, make sure we get one sample nonetheless... if (Rate == 0.0f && !SeekTargetTime.IsValid()) { // Do this once after open / seek... if (LastVideoSampleProcessedTimeRange.IsEmpty()) { // Use the video sample timestamp for simplicity (although we otherwise sync with audio timestamps) FMediaTimeStamp TimeStamp; if (Player->GetSamples().PeekVideoSampleTime(TimeStamp)) { TimeRange = TRange(TimeStamp, TimeStamp + DeltaTime); return !TimeRange.IsEmpty(); } } } } // No timing info available, no time range available, no samples to process return false; } FMediaTimeStamp EstAudioTimeAtFrameStart; double Now = FPlatformTime::Seconds(); if (!bPurgeSampleRelated) { // Normal estimation relative to current frame start... // (on gamethread operation) check(IsInGameThread() || IsInSlateThread()); double AgeOfFrameStart = Now - MediaModule->GetFrameStartTime(); double AgeOfAudioTime = Now - AudioTime.SampledAtTime; if (AgeOfFrameStart >= 0.0 && AgeOfFrameStart <= kMaxTimeSinceFrameStart && AgeOfAudioTime >= 0.0 && AgeOfAudioTime <= kMaxTimeSinceAudioTimeSampling) { // All realtime timestamps seem in sane ranges - we most likely did not have a lengthy interruption (suspended / debugging step) EstAudioTimeAtFrameStart = AudioTime.TimeStamp + FTimespan::FromSeconds((MediaModule->GetFrameStartTime() - AudioTime.SampledAtTime) * Rate); } else { // Realtime timestamps seem wonky. Proceed without them (worse estimation quality) EstAudioTimeAtFrameStart = AudioTime.TimeStamp; } } else { // Do not use frame start reference -> we compute relative to "now" // (for use off gamethread) EstAudioTimeAtFrameStart = AudioTime.TimeStamp + FTimespan::FromSeconds((Now - AudioTime.SampledAtTime) * Rate); } // Are we paused? if (Rate == 0.0f) { // Yes. We need to fetch a frame for the current display frame - once. Asking over and over until we get one... if (LastVideoSampleProcessedTimeRange.IsEmpty()) { // We simply fake the rate to the last non-zero or 1.0 to fetch a frame fitting the time frame representing the whole current frame Rate = (LastRate == 0.0f) ? 1.0f : LastRate; } } TimeRange = TRange(EstAudioTimeAtFrameStart, EstAudioTimeAtFrameStart + DeltaTime * FGenericPlatformMath::Abs(Rate)); } else { // // No Audio (no data and/or no sink) // if (!BlockOnRange.IsSet()) { // // Internal clock (DT based) // // Do we now have a current timestamp estimation? if (!NextEstVideoTimeAtFrameStart.IsValid()) { // No timing info available, no time range available, no samples to process return false; } else { // Yes. Setup current time range & advance time estimation... // Are we paused? if (Rate == 0.0f) { // Yes. We need to fetch a frame for the current display frame - once. Asking over and over until we get one... if (LastVideoSampleProcessedTimeRange.IsEmpty()) { // We simply fake the rate to the last non-zero or 1.0 to fetch a frame fitting the time frame representing the whole current frame Rate = (LastRate == 0.0f) ? 1.0f : LastRate; } } TimeRange = TRange(NextEstVideoTimeAtFrameStart.TimeStamp, NextEstVideoTimeAtFrameStart.TimeStamp + DeltaTime * FGenericPlatformMath::Abs(Rate)); } } else { // // External clock delivers time-range // (for now we just use the blocking time range as this clock type is solely used in that case) // TimeRange = GetAdjustedBlockOnRange(); } } if (TimeRange.IsEmpty()) { return false; } const FTimespan Duration = Player->GetControls().GetDuration(); TRange ActiveRange = GetActivePlaybackRange(); // We need a valid duration for the next steps (we may not have one e.g. for live material) if (IsDurationValidAndFinite(Duration)) { const FTimespan ActiveRangeDuration = ActiveRange.GetUpperBoundValue() - ActiveRange.GetLowerBoundValue(); // If we are looping we check to prepare proper ranges should we wrap around either end of the media... // (we do not clamp in the non-looping case as the rest of the code should deal with that fine) if (Player->GetControls().IsLooping()) { FTimespan WrappedStart = WrappedModulo(TimeRange.GetLowerBoundValue().Time - ActiveRange.GetLowerBoundValue(), ActiveRangeDuration) + ActiveRange.GetLowerBoundValue(); FTimespan WrappedEnd = WrappedModulo(TimeRange.GetUpperBoundValue().Time - ActiveRange.GetLowerBoundValue(), ActiveRangeDuration) + ActiveRange.GetLowerBoundValue(); if (WrappedStart > WrappedEnd) { if (WrappedStart != TimeRange.GetLowerBoundValue().Time) { TimeRange.SetLowerBoundValue(FMediaTimeStamp(WrappedStart, TimeRange.GetLowerBoundValue().GetSequenceIndex(), TimeRange.GetLowerBoundValue().GetLoopIndex() -1)); } if (WrappedEnd != TimeRange.GetUpperBoundValue().Time) { TimeRange.SetUpperBoundValue(FMediaTimeStamp(WrappedEnd, TimeRange.GetUpperBoundValue().GetSequenceIndex(), TimeRange.GetUpperBoundValue().GetLoopIndex() + 1)); } } } else { TimeRange.SetLowerBoundValue(FMediaTimeStamp(FMath::Clamp(TimeRange.GetLowerBoundValue().Time, ActiveRange.GetLowerBoundValue(), ActiveRange.GetUpperBoundValue()), TimeRange.GetLowerBoundValue().GetSequenceIndex(), TimeRange.GetLowerBoundValue().GetLoopIndex())); TimeRange.SetUpperBoundValue(FMediaTimeStamp(FMath::Clamp(TimeRange.GetUpperBoundValue().Time, ActiveRange.GetLowerBoundValue(), ActiveRange.GetUpperBoundValue()), TimeRange.GetUpperBoundValue().GetSequenceIndex(), TimeRange.GetUpperBoundValue().GetLoopIndex())); } } return !TimeRange.IsEmpty(); } TRange FMediaPlayerFacade::GetAdjustedBlockOnRange() const { TRange TimeRange = BlockOnRange.GetRange(); return TimeRange; } /* FMediaPlayerFacade implementation *****************************************************************************/ void FMediaPlayerFacade::ProcessAudioSamples(IMediaSamples& Samples, const TRange& TimeRange) { TSharedPtr Sample; check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); // For V2 we basically expect to get no timerange at all: totally open // (we just have it around to be compatible / use older code that expects it) check(TimeRange.GetLowerBound().IsOpen() && TimeRange.GetUpperBound().IsOpen()); // // "Modern" 1-Audio-Sink-Only case (aka: we only feed the primary sink) // if (TSharedPtr< FMediaAudioSampleSink, ESPMode::ThreadSafe> PinnedPrimaryAudioSink = PrimaryAudioSink.Pin()) { while(PinnedPrimaryAudioSink->CanAcceptSamples(1)) { if (!Samples.FetchAudio(TimeRange, Sample)) { break; } else if (!Sample.IsValid()) { continue; } { FScopeLock Lock(&LastTimeValuesCS); LastAudioSampleProcessedTime.TimeStamp = FMediaTimeStamp(Sample->GetTime()); LastAudioSampleProcessedTime.SampledAtTime = FPlatformTime::Seconds(); } PinnedPrimaryAudioSink->Enqueue(Sample.ToSharedRef()); } } else { // Do we have video playback? if (HaveVideoPlayback()) { TRange TempRange; // We got video and audio, but no audio sink - throw away anything up to video playback time... // (rough estimate, as this is off-gamethread; but better than throwing things out with no throttling at all) { bool bReverse = (CurrentRate < 0.0f); FScopeLock Lock(&LastTimeValuesCS); if (!bReverse) { TempRange.SetUpperBound(CurrentFrameVideoTimeStamp); } else { TempRange.SetLowerBound(CurrentFrameVideoTimeStamp); } } while(Samples.FetchAudio(TempRange, Sample)) { } } else { // No Video and no primary audio sink: we throw all away (sub-optimal as it will keep audio decoding busy; but this should be an edge case) while(Samples.FetchAudio(TimeRange, Sample)) { } } } } bool FMediaPlayerFacade::IsVideoSampleStillGood(const TRange& LastSampleTimeRange, const TRange& TimeRange, bool bReverse) const { // If we have no valid time range or a seek is in progress we assume the current frame can be considered "done" in any case if (!TimeRange.IsEmpty() && !SeekTargetTime.IsValid() && !LastSampleTimeRange.IsEmpty()) { // This is not the case: check more detailed! // This better be true at all times check(LastSampleTimeRange.GetLowerBoundValue().GetIndexValue() == LastSampleTimeRange.GetUpperBoundValue().GetIndexValue()); // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Remap all values so we can assume all of them to be in a single "sequence index range" so the math doesn't get too unruly below. // For that the sequence index must not have changed as we otherwise cannot assume any valid frame. if (TimeRange.GetLowerBoundValue().GetSequenceIndex() != TimeRange.GetUpperBoundValue().GetSequenceIndex()) { return false; } FTimespan Duration = Player->GetControls().GetDuration(); TRange TimeRange0; const int32 LoopIndex0 = TimeRange.GetLowerBoundValue().GetLoopIndex(); const int32 LoopIndex1 = TimeRange.GetUpperBoundValue().GetLoopIndex(); int32 RefLoopIndex = LoopIndex0; // If we encounter a loop we need to check if we can "unroll" it... if (LoopIndex0 != LoopIndex1) { // We only should get here with a looping player that knows its duration check(Player->GetControls().IsLooping()); check(IsDurationValidAndFinite(Duration)); // Compute how many loops and change the range into one "unrolled" one as indicated by the playback direction... int32 LoopIdxDiff = LoopIndex1 - LoopIndex0; // Note: this will be positive even with reverse playback as the orientation of the range will no change check(LoopIdxDiff > 0); double DurationD = Duration.GetTotalSeconds(); if (!bReverse) { TimeRange0 = TRange(FMediaTimeStamp(TimeRange.GetLowerBoundValue().Time, 0), FMediaTimeStamp(TimeRange.GetUpperBoundValue().Time + FTimespan::FromSeconds(LoopIdxDiff * DurationD), 0)); } else { TimeRange0 = TRange(FMediaTimeStamp(TimeRange.GetLowerBoundValue().Time - FTimespan::FromSeconds(LoopIdxDiff * DurationD), 0), FMediaTimeStamp(TimeRange.GetUpperBoundValue().Time, 0)); RefLoopIndex = LoopIndex1; } } else { // Simple case, just bring everything down to "zero sequence index" for ease of processing below... TimeRange0 = TRange(FMediaTimeStamp(TimeRange.GetLowerBoundValue().Time, 0, 0), FMediaTimeStamp(TimeRange.GetUpperBoundValue().Time, 0, 0)); // Is looping off? if (!Player->GetControls().IsLooping()) { // Yes. We clamp the range to the duration of the video to avoid looking at non-existent "next" frames... (unless we have no duration) if (IsDurationValidAndFinite(Duration)) { const TRange ActiveRange = GetActivePlaybackRange(); TimeRange0 = TRange::Intersection(TimeRange0, TRange(FMediaTimeStamp(ActiveRange.GetLowerBoundValue(), 0), FMediaTimeStamp(ActiveRange.GetUpperBoundValue(), 0))); } } } // Map the last sample's time range to the same "sequence index" range as the time range // (note: for Live streams that do not have any set duration all this will not change the timerange - just as needed) const int32 LastSampleLoopDiff = LastSampleTimeRange.GetLowerBoundValue().GetLoopIndex() - RefLoopIndex; FTimespan TimeOffset = IsDurationValidAndFinite(Duration) ? Duration * LastSampleLoopDiff : FTimespan::Zero(); TRange LastSampleTimeRange0(FMediaTimeStamp(LastSampleTimeRange.GetLowerBoundValue().Time + TimeOffset, 0, 0), FMediaTimeStamp(LastSampleTimeRange.GetUpperBoundValue().Time + TimeOffset, 0, 0)); // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Now we can begin the checks with all time ranges mapped back to "zero sequence index" // Is the sample time range ahead of the given time range? // (did the range move in an unexpected way?) if (!bReverse ? TimeRange0.GetUpperBoundValue() <= LastSampleTimeRange0.GetLowerBoundValue() : TimeRange0.GetLowerBoundValue() >= LastSampleTimeRange0.GetUpperBoundValue()) { // We simply let the last sample stay around... return true; } // Is the sample time range at all still valid? if (LastSampleTimeRange0.Overlaps(TimeRange0)) { // Yes. Assuming we could get more samples (of the same type) from the player, would the next one be "better"? // (we assume samples of equal length) // Compute the "theoretical" next sample range... TRange NextSampleTimeRange = !bReverse ? TRange(LastSampleTimeRange0.GetUpperBoundValue(), LastSampleTimeRange0.GetUpperBoundValue() + LastSampleTimeRange0.Size().Time) : TRange(LastSampleTimeRange0.GetLowerBoundValue() - LastSampleTimeRange0.Size().Time, LastSampleTimeRange0.GetLowerBoundValue()); // Note: Loops (or the end of the time line in non-looping setups) // // - We could check for them and generate proper changes to the sequence index // - Doing this would leave us with quite complex setups to compute the coverage // - We opt for a cleaner, simpler approach: as we are NOT interested in proper PTS values here, we can safely work with an "infinite" time line when computing any overlaps, coverage and such // (note: we DO need to restrict the range to the actual media duration if not looping - the code above does this) // // --> we simply keep what we compute above! // // Compute which one is larger inside the current range... int64 LastSampleCoverage = TRange::Intersection(TimeRange0, LastSampleTimeRange0).Size().Time.GetTicks(); int64 NextSampleCoverage = TRange::Intersection(TimeRange0, NextSampleTimeRange).Size().Time.GetTicks(); // A new one is only desirable if it's BETTER than the current one if (LastSampleCoverage >= NextSampleCoverage) { // Last one we returned is still good. No new one needed... return true; } } } return false; } bool FMediaPlayerFacade::ProcessVideoSamples(IMediaSamples& Samples, const TRange& TimeRange) { if (!Player.IsValid()) { // Nothing to do, but in a sense: "successful"... return true; } // This is not to be used with V1 timing check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); // We expect a fully closed range or we assume: nothing to do... check(TimeRange.GetLowerBound().IsClosed() && TimeRange.GetUpperBound().IsClosed()); TSharedPtr Sample; if (!Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::AlwaysPullNewestVideoFrame)) { // // Normal playback with timing control provided by MediaFramework // const bool bReverse = (GetUnpausedRate() < 0.0f); if (IsVideoSampleStillGood(LastVideoSampleProcessedTimeRange, TimeRange, bReverse)) { // We got all the samples we need. Processing was successful... return true; } switch(Samples.FetchBestVideoSampleForTimeRange(TimeRange, Sample, bReverse, BlockOnRange.IsSet())) { case IMediaSamples::EFetchBestSampleResult::Ok: { break; } case IMediaSamples::EFetchBestSampleResult::NoSample: { break; } case IMediaSamples::EFetchBestSampleResult::PurgedToEmpty: { // When there is no audio to sync to then we are extrapolating the next expected video timestamp // from the last plus the elapsed deltatime, which may overshoot the next decoder output. // In this case we resynchronize the timestamp to the next available video frame. if (!HaveAudioPlayback()) { NextEstVideoTimeAtFrameStart.Invalidate(); } break; } case IMediaSamples::EFetchBestSampleResult::NotSupported: { // // Fallback for players supporting V2 timing, but do not supply FetchBestVideoSampleForTimeRange() due to some // custom implementation of IMediaSamples (here to ease adoption of the new timing code - eventually should go away) // // Find newest sample that satisfies the time range // (the FetchXYZ() code does not work well with a lower range limit at all - we ask for a "up to" type range instead // and limit the other side of the range in code here to not change the older logic & possibly cause trouble in old code) TRange TempRange = bReverse ? TRange::AtLeast(TimeRange.GetUpperBoundValue()) : TRange::AtMost(TimeRange.GetUpperBoundValue()); while(Samples.FetchVideo(TempRange, Sample)) { } if (Sample.IsValid() && ((!bReverse && ((Sample->GetTime() + Sample->GetDuration()) > TimeRange.GetLowerBoundValue())) || (bReverse && ((Sample->GetTime() - Sample->GetDuration()) < TimeRange.GetLowerBoundValue())))) { // Sample is good - nothing more to do here } else { Sample.Reset(); } break; } } } else { // // Use newest video frame available at all times (no Mediaframework timing control) // TRange TempRange; // fully open range while(Samples.FetchVideo(TempRange, Sample)) { } } // Any sample? if (Sample.IsValid()) { // Yes, deliver it and update state... FMediaTimeStamp SampleTime = Sample->GetTime(); TRange SampleTimeRange(SampleTime, SampleTime + Sample->GetDuration()); // Enqueue the sample to render // (we use a queue to stay compatible with existing structure and older sinks - new sinks will read this single entry right away on the gamethread // and pass it along to rendering outside the queue) bool bOk = VideoSampleSinks.Enqueue(Sample.ToSharedRef()); check(bOk); { FScopeLock Lock(&LastTimeValuesCS); CurrentFrameVideoDisplayTimeStamp = CurrentFrameVideoTimeStamp = SampleTimeRange.GetLowerBoundValue(); LastVideoSampleProcessedTimeRange = SampleTimeRange; MostRecentlyDeliveredVideoFrameTimecode = Sample->GetTimecode(); } UpdateSeekStatus(&CurrentFrameVideoTimeStamp); return true; } return false; } void FMediaPlayerFacade::ProcessCaptionSamples(IMediaSamples& Samples, const TRange& TimeRange) { check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); TSharedPtr Sample; // Seek in progress? if (SeekTargetTime.IsValid()) { // Yes. Fetch (and discard) all samples up to the seek target time... // (we only throw out samples from prior sequence indices to make sure we do not swallow any audio from overlapping samples) auto DiscardRange = TRange(FMediaTimeStamp(0), FMediaTimeStamp((CurrentRate >= 0.0f) ? FTimespan::Zero() : FTimespan::MaxValue(), SeekTargetTime.GetSequenceIndex(), SeekTargetTime.GetLoopIndex())); Samples.DiscardCaptionSamples(DiscardRange, GetUnpausedRate() < 0.0f); } else { while(Samples.FetchCaption(TimeRange, Sample)) { if (Sample.IsValid() && !CaptionSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Caption sample sink overflow")); #endif } } } } void FMediaPlayerFacade::ProcessSubtitleSamples(IMediaSamples& Samples, const TRange& TimeRange) { check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); TSharedPtr Sample; // Seek in progress? if (SeekTargetTime.IsValid()) { // Yes. Fetch (and discard) all samples up to the seek target time... // (we only throw out samples from prior sequence indices to make sure we do not swallow any audio from overlapping samples) auto DiscardRange = TRange(FMediaTimeStamp(0), FMediaTimeStamp((CurrentRate >= 0.0f) ? FTimespan::Zero() : FTimespan::MaxValue(), SeekTargetTime.GetSequenceIndex(), SeekTargetTime.GetLoopIndex())); Samples.DiscardSubtitleSamples(DiscardRange, GetUnpausedRate() < 0.0f); } else { while(Samples.FetchSubtitle(TimeRange, Sample)) { //UE_LOG(LogMediaUtils, Display, TEXT("Subtitle @%.3f: %s"), Sample->GetTime().Time.GetTotalSeconds(), *Sample->GetText().ToString()); if (Sample.IsValid() && !SubtitleSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Subtitle sample sink overflow")); #endif } } } } void FMediaPlayerFacade::ProcessMetadataSamples(IMediaSamples& Samples, const TRange& TimeRange) { check(Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); TSharedPtr Sample; // Seek in progress? if (SeekTargetTime.IsValid()) { // Yes. Fetch (and discard) all samples up to the seek target time... // (we only throw out samples from prior sequence indices to make sure we do not swallow any audio from overlapping samples) auto DiscardRange = TRange(FMediaTimeStamp(0), FMediaTimeStamp((CurrentRate >= 0.0f) ? FTimespan::Zero() : FTimespan::MaxValue(), SeekTargetTime.GetSequenceIndex(), SeekTargetTime.GetLoopIndex())); Samples.DiscardMetadataSamples(DiscardRange, GetUnpausedRate() < 0.0f); } else { while(Samples.FetchMetadata(TimeRange, Sample)) { if (Sample.IsValid() && !MetadataSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Metadata sample sink overflow")); #endif } } } } void FMediaPlayerFacade::ProcessAudioSamplesV1(IMediaSamples& Samples, TRange TimeRange) { TSharedPtr Sample; while(Samples.FetchAudio(TimeRange, Sample)) { if (!Sample.IsValid()) { continue; } if (!AudioSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Audio sample sink overflow")); #endif } else { FScopeLock Lock(&LastTimeValuesCS); LastAudioSampleProcessedTime.TimeStamp = Sample->GetTime(); LastAudioSampleProcessedTime.SampledAtTime = FPlatformTime::Seconds(); } } } void FMediaPlayerFacade::ProcessVideoSamplesV1(IMediaSamples& Samples, TRange TimeRange) { // This is not to be used with V2 timing check(!Player->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); TSharedPtr Sample; while(Samples.FetchVideo(TimeRange, Sample)) { if (!Sample.IsValid()) { continue; } { FScopeLock Lock(&LastTimeValuesCS); CurrentFrameVideoDisplayTimeStamp = CurrentFrameVideoTimeStamp = Sample->GetTime(); } UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Fetched video sample %s"), *Sample->GetTime().Time.ToString(TEXT("%h:%m:%s.%t"))); if (VideoSampleSinks.Enqueue(Sample.ToSharedRef())) { if (CurrentRate >= 0.0f) { NextVideoSampleTime = Sample->GetTime().Time + Sample->GetDuration(); UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Next video sample time %s"), *NextVideoSampleTime.ToString(TEXT("%h:%m:%s.%t"))); } } else { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Video sample sink overflow")); #endif } } } void FMediaPlayerFacade::ProcessCaptionSamplesV1(IMediaSamples& Samples, TRange TimeRange) { TSharedPtr Sample; while(Samples.FetchCaption(TimeRange, Sample)) { if (Sample.IsValid() && !CaptionSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Caption sample sink overflow")); #endif } } } void FMediaPlayerFacade::ProcessSubtitleSamplesV1(IMediaSamples& Samples, TRange TimeRange) { TSharedPtr Sample; while(Samples.FetchSubtitle(TimeRange, Sample)) { if (Sample.IsValid() && !SubtitleSampleSinks.Enqueue(Sample.ToSharedRef())) { FString Caption = Sample->GetText().ToString(); //UE_LOG(LogMediaUtils, Log, TEXT("New caption @%.3f: %s"), Sample->GetTime().Time.GetTotalSeconds(), *Caption); #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Subtitle sample sink overflow")); #endif } } } void FMediaPlayerFacade::ProcessMetadataSamplesV1(IMediaSamples& Samples, TRange TimeRange) { TSharedPtr Sample; while(Samples.FetchMetadata(TimeRange, Sample)) { if (Sample.IsValid() && !MetadataSampleSinks.Enqueue(Sample.ToSharedRef())) { #if MEDIAPLAYERFACADE_TRACE_SINKOVERFLOWS UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Metadata sample sink overflow")); #endif } } } /* IMediaEventSink interface *****************************************************************************/ void FMediaPlayerFacade::ReceiveMediaEvent(EMediaEvent Event) { if (Event >= EMediaEvent::Internal_Start) { switch (Event) { case EMediaEvent::Internal_PurgeVideoSamplesHint: { /* Sent by some media players to ask to purge older samples from the video output queue. This is done to ensure that, even if the game thread is stalled and the facade is not being ticked regularly where it would perform this task by passing frames from the queue to the sink, frames that have passed the point where they should have been sent to the sink will not clog the queue. The player cannot perform this task on its own because it does not know the current precise playback position. Here we need to handle only everything not audio because audio is pulled by the audio thread and not the gamethread, so it can never stall. */ TSharedPtr CurrentPlayer = Player; if (!CurrentPlayer.IsValid()) { return; } // We only support this for V2 timing players check(CurrentPlayer->GetPlayerFeatureFlag(IMediaPlayer::EFeatureFlag::UsePlaybackTimingV2)); // Only do this if we do not block on time ranges if (BlockOnRange.IsSet()) { // We do not purge as we do not need max perf, but max reliability to actually get certain frames return; } const float Rate = CurrentRate; if (Rate == 0.0f) { return; } // Get current playback time // (Note: the delta time is entirely synthetic - we do not pass zero to avoid an empty range, but we do not look far into the future either // -> after all: we are mainly focused on purging samples up to the current time // Remarks: // - this version does not take any estimations from any frame start into account as this is entirely async to the main thread // - video streams with no audio content will be played using the UE DeltaTime -> so if that stops, the progress of the video stops! // -> hence we will not see (other then one initial purge) any purging of samples here! // ) TRange TimeRange; if (!GetCurrentPlaybackTimeRange(TimeRange, Rate, FTimespan::FromMilliseconds(kOutdatedSamplePurgeRange), true)) { return; } bool bReverse = (Rate < 0.0f); const float RateFactor = (Rate != 0.0f) ? (1.0f / Rate) : 1.0f; // Don't purge frames if the queue is small (to avoid purging if players deliver frames late persistently) uint32 NumPurged = 0; if (CurrentPlayer->GetSamples().NumVideoSamples() >= kMinFramesInVideoQueueToPurge) { NumPurged = CurrentPlayer->GetSamples().PurgeOutdatedVideoSamples(TimeRange.GetLowerBoundValue(), bReverse, FTimespan::FromSeconds(kOutdatedVideoSamplesTolerance * RateFactor)); } SET_DWORD_STAT(STAT_MediaUtils_FacadeNumPurgedVideoSamples, NumPurged); INC_DWORD_STAT_BY(STAT_MediaUtils_FacadeTotalPurgedVideoSamples, NumPurged); // Take the opportunity to also purge any samples related to video samples directly (and evaluated on the game thread) // Captions... NumPurged = 0; if (CurrentPlayer->GetSamples().NumCaptionSamples() >= kMinFramesInCaptionQueueToPurge) { NumPurged = CurrentPlayer->GetSamples().PurgeOutdatedCaptionSamples(TimeRange.GetLowerBoundValue(), bReverse, FTimespan::FromSeconds(kOutdatedSubtitleSamplesTolerance * RateFactor)); } SET_DWORD_STAT(STAT_MediaUtils_FacadeNumPurgedSubtitleSamples, NumPurged); INC_DWORD_STAT_BY(STAT_MediaUtils_FacadeTotalPurgedSubtitleSamples, NumPurged); // Subtitles... NumPurged = 0; if (CurrentPlayer->GetSamples().NumSubtitleSamples() >= kMinFramesInSubtitleQueueToPurge) { NumPurged = CurrentPlayer->GetSamples().PurgeOutdatedSubtitleSamples(TimeRange.GetLowerBoundValue(), bReverse, FTimespan::FromSeconds(kOutdatedSubtitleSamplesTolerance * RateFactor)); } SET_DWORD_STAT(STAT_MediaUtils_FacadeNumPurgedCaptionSamples, NumPurged); INC_DWORD_STAT_BY(STAT_MediaUtils_FacadeTotalPurgedCaptionSamples, NumPurged); break; } case EMediaEvent::Internal_VideoSamplesAvailable: { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Video samples ARE available")); VideoSampleAvailability = 1; break; } case EMediaEvent::Internal_VideoSamplesUnavailable: { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Video samples are NOT available")); VideoSampleAvailability = 0; break; } case EMediaEvent::Internal_AudioSamplesAvailable: { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Audio samples ARE available")); AudioSampleAvailability = 1; break; } case EMediaEvent::Internal_AudioSamplesUnavailable: { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Audio samples are NOT available")); AudioSampleAvailability = 0; break; } default: { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Received media event %s"), *MediaUtils::EventToString(Event)); break; } } } else { UE_LOG(LogMediaUtils, VeryVerbose, TEXT("PlayerFacade: Received media event %s"), *MediaUtils::EventToString(Event)); QueuedEvents.Enqueue(Event); } } void FMediaPlayerFacade::ReInit() { // We leave the registered sinks and delegates alone { FScopeLock lock(&CriticalSection); BlockOnRange.Reset(); BlockOnRangeDisabled = false; CurrentUrl.Empty(); LastRate = 0.0f; CurrentRate = 0.0f; bHaveActiveAudio = false; VideoSampleAvailability = -1; AudioSampleAvailability = -1; NextVideoSampleTime = FTimespan::Zero(); } { FScopeLock lock(&LastTimeValuesCS); LastAudioRenderedSampleTime.Invalidate(); LastAudioSampleProcessedTime.Invalidate(); LastVideoSampleProcessedTimeRange = TRange::Empty(); CurrentFrameAudioTimeStamp.Invalidate(); CurrentFrameVideoTimeStamp.Invalidate(); CurrentFrameVideoDisplayTimeStamp.Invalidate(); NextEstVideoTimeAtFrameStart.Invalidate(); MostRecentlyDeliveredVideoFrameTimecode.Reset(); SeekTargetTime.Invalidate(); NextSeekTime.Reset(); NextSequenceIndex.Reset(); } }