Files
UnrealEngine/Engine/Extras/VirtualProduction/LiveLinkVCAM/vcam/VideoViewController.swift
2025-05-18 13:04:45 +08:00

575 lines
23 KiB
Swift

//
// ViewController.swift
// vcam
//
// Created by Brian Smith on 8/8/20.
// Copyright Epic Games, Inc. All Rights Reserved.
//
import UIKit
import MetalKit
import ARKit
import Easing
import GameController
class VideoViewController : BaseViewController {
var demoMode : Bool {
liveLink == nil
}
weak var liveLink : LiveLinkProvider?
// Required public for reconnection picker
var pickerData: [String] = [String]()
var selectedStreamer: String = ""
private var displayLink: CADisplayLink?
private var refreshRateHint : CADisplayLink?
private var lastTimestamp: CFTimeInterval = 0.0
private var arSession : ARSession?
private var arCoachingView : ARCoachingOverlayView?
@IBOutlet weak var renderView : UIView!
@IBOutlet weak var headerView : HeaderView!
@IBOutlet weak var headerViewTopConstraint : NSLayoutConstraint!
private var headerViewY : CGFloat = 0
private var headerViewHeight : CGFloat = 0
private var headerViewTopConstraintStartValue : CGFloat = 0
private var headerPanGestureRecognizer : UIPanGestureRecognizer!
private var headerPullDownGestureRecognizer : UIScreenEdgePanGestureRecognizer!
private var statsTimer : Timer?
private var showStats : Bool = false
private var gameControllerSnapshot : GCController?
// streamingConnection and gameController are passed from StartViewController
weak var streamingConnection : StreamingConnection?
weak var gameController : GCController? {
didSet {
if let gc = gameController {
gc.extendedGamepad?.dpad.up.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.dpad.down.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.dpad.left.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.dpad.right.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonA.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonB.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonX.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonY.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.leftShoulder.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.rightShoulder.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.leftTrigger.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.rightTrigger.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.leftThumbstickButton?.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.rightThumbstickButton?.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonOptions?.pressedChangedHandler = self.buttonValueChanged
gc.extendedGamepad?.buttonMenu.pressedChangedHandler = self.buttonValueChanged
}
}
}
private var gamepads: [GCControllerPlayerIndex : Gamepad] = [:];
@IBOutlet weak var arView : ARSCNView!
@IBOutlet weak var reconnectingBlurView : UIVisualEffectView!
override var prefersHomeIndicatorAutoHidden: Bool {
return true
}
override var prefersStatusBarHidden: Bool {
return true
}
override var preferredScreenEdgesDeferringSystemGestures: UIRectEdge {
return [.top]
}
@objc func applicationDidBecomeActive(notification: NSNotification) {
reconnect()
}
@objc func applicationDidEnterBackground(notification: NSNotification) {
disconnect()
}
@objc func controllerDidConnect(notification: NSNotification) {
guard let gc = notification.object as? GCController else { return }
self.controllerConnected(gamepad: gc)
}
@objc func controllerDidDisconnect(notification: NSNotification) {
guard let gc = notification.object as? GCController else { return }
self.controllerDisconnected(gamepad: gc)
}
// Constructor (before view is even loaded)
required init?(coder: NSCoder) {
super.init(coder: coder)
setupObservers()
setupARSession()
}
// Destructor (called when view controller is destroyed)
deinit {
Log.info("VideoViewController destructed.")
self.headerView = nil
}
func setupObservers(){
// According to: https://stackoverflow.com/a/40339926
// There is no need to remove observers because they are captured weakly and automatically removed (as long as they are not using closure blocks)
NotificationCenter.default.addObserver(self, selector: #selector(applicationDidBecomeActive), name: UIApplication.willEnterForegroundNotification, object: nil)
NotificationCenter.default.addObserver(self, selector: #selector(applicationDidEnterBackground), name: UIApplication.didEnterBackgroundNotification, object: nil)
// Add notifications for when new controllers connect or disconnect
NotificationCenter.default.addObserver(self, selector: #selector(controllerDidConnect), name: .GCControllerDidConnect, object: nil)
NotificationCenter.default.addObserver(self, selector: #selector(controllerDidDisconnect), name: .GCControllerDidDisconnect, object: nil)
}
func setupGestureRecognizers(){
self.headerPanGestureRecognizer = UIPanGestureRecognizer(target: self, action: #selector(handleHeaderPanGesture))
self.headerView.addGestureRecognizer(self.headerPanGestureRecognizer)
self.headerPullDownGestureRecognizer = UIScreenEdgePanGestureRecognizer(target: self, action: #selector(handleHeaderPanGesture))
self.headerPullDownGestureRecognizer.edges = [ .top ]
self.view.addGestureRecognizer(self.headerPullDownGestureRecognizer)
self.headerPanGestureRecognizer.require(toFail: headerPullDownGestureRecognizer)
}
func removeGestureRecognizers(){
self.headerView.removeGestureRecognizer(self.headerPanGestureRecognizer)
self.view.removeGestureRecognizer(self.headerPullDownGestureRecognizer)
}
func setupARSession(){
arSession = ARSession()
arSession?.delegate = self
}
func startARSession(){
// Create a ARKit tracking config for purely tracking device transform (we don't care about the computer vision features)
let config = ARPositionalTrackingConfiguration()
config.worldAlignment = .gravity
config.planeDetection = []
config.isLightEstimationEnabled = false
config.providesAudioData = false
if self.arSession == nil {
setupARSession()
}
self.arSession?.run(config)
}
override func viewDidLoad() {
super.viewDidLoad()
self.headerView.start()
headerViewHeight = self.headerView.frame.height
// The AR video feed & CG objects only exist in demo mode.
// In normal operation, they are replaced by a live feed of the Unreal Engine rendering, with the virtual camera's position determined by the device & ARKit.
//self.demoModeBlurView.isHidden = !self.demoMode
showReconnecting(false, animated: false)
// setup gesture recognition has to start here because it references view that need to have loaded
self.setupGestureRecognizers()
// Start running AR session
self.startARSession()
// Attach an empty view where the relevant stream connection can setup its own view once this is set
self.streamingConnection?.renderView = self.renderView
if let existingARCoachingView = self.arCoachingView {
existingARCoachingView.removeFromSuperview()
}
// Insert coaching overlay into subview
let coachingOverlayView = ARCoachingOverlayView()
self.view.insertSubview(coachingOverlayView, belowSubview: self.reconnectingBlurView)
coachingOverlayView.layout(.left, .right, .top, .bottom, to: self.arView)
coachingOverlayView.goal = .tracking
coachingOverlayView.session = self.arSession
coachingOverlayView.activatesAutomatically = true
coachingOverlayView.delegate = self
self.arCoachingView = coachingOverlayView
statsTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true, block: { t in
self.updateStreamingStats()
})
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
// Ensure we do not go into sleep mode while this view is active
UIApplication.shared.isIdleTimerDisabled = true
// Check for already connected controllers
for gc in GCController.controllers() {
self.controllerConnected(gamepad: gc)
}
// Todo: Could dynamically adjust this
self.setRefreshRateFps(fps: 60)
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
// Now this view is going away we can let the device go to sleep if there is not touch input
UIApplication.shared.isIdleTimerDisabled = false
// If view is going away we will remove out refresh rate hint so battery usage can return to normal values
self.resetRefreshRateFps()
// Clear stats timer
self.statsTimer?.invalidate()
// pause ar session when view goes away
self.arSession?.pause()
// remove the gesture recognizers as this view will not longer be shown
self.removeGestureRecognizers()
// disconnect from streaming if we haven't already
self.disconnect()
}
@objc func refreshRateCallback(_ displayLink: CADisplayLink) {
// Todo: We could show the refresh rate in the UI from here?
//let deltaTime = displayLink.timestamp - self.lastTimestamp
//Log.info(String(deltaTime))
//let workingTime = displayLink.targetTimestamp - CACurrentMediaTime()
//Log.info(String(workingTime))
self.lastTimestamp = displayLink.timestamp
}
func setRefreshRateFps(fps: Int) {
self.resetRefreshRateFps()
// Attempt to force the display refresh rate to the 60-120hz range for WebRTC video streaming (it seems iOS does not auto detect the rate of received frames and adjust)
self.refreshRateHint = CADisplayLink(target: self, selector: #selector(refreshRateCallback))
self.refreshRateHint?.preferredFrameRateRange = CAFrameRateRange(minimum: Float(fps), maximum: Float(fps), preferred: Float(fps))
self.refreshRateHint?.add(to: .main, forMode: .common)
for subview in self.renderView.subviews {
if let webrtcView = subview as? WebRTCView {
webrtcView.videoView?.setPreferredFramerate(fps: fps)
}
}
}
func resetRefreshRateFps() {
self.refreshRateHint?.remove(from: .main, forMode: .common)
self.refreshRateHint?.invalidate()
self.refreshRateHint = nil
}
func showReconnecting(_ visible : Bool, animated: Bool) {
if (self.reconnectingBlurView.effect != nil && visible) ||
(self.reconnectingBlurView.effect == nil && !visible) {
return
}
UIView.animate(withDuration: 0.0, animations: {
self.reconnectingBlurView.effect = visible ? UIBlurEffect(style: UIBlurEffect.Style.dark) : nil
})
if visible {
showConnectingAlertView(mode : .reconnecting) {
self.exit()
}
} else {
hideConnectingAlertView() {}
}
}
func exit() {
disconnect()
}
func reconnect() {
showReconnecting(true, animated: true)
self.streamingConnection?.reconnect()
}
func disconnect() {
// Tell UE that the controllers connected to this device are no longer used
for gc in GCController.controllers() {
self.controllerDisconnected(gamepad: gc)
}
self.streamingConnection?.disconnect()
// Clear the delegate on the streaming connection (which is a reference to this view controller)
self.streamingConnection?.delegate = nil
// Clear the streaming connection itself
self.streamingConnection = nil
// Dismiss this UIViewController and return to the presenting view that segued to here
self.presentingViewController?.dismiss(animated: true, completion: nil)
}
func updateStreamingStats() {
var str = ""
if let sz = self.streamingConnection?.videoSize {
str += "\(Int(sz.width))x\(Int(sz.height))"
}
if let stats = self.streamingConnection?.stats {
if let bps = stats.bytesPerSecond {
if !str.isEmpty {
str += ""
}
str += ByteCountFormatter().string(fromByteCount: Int64(bps)) + "/sec"
}
if let fps = stats.framesPerSecond {
if !str.isEmpty {
str += ""
}
str += "\(fps) fps"
}
}
self.headerView.stats = str
}
@objc func handleHeaderPanGesture(_ gesture : UIGestureRecognizer) {
guard let panGesture = gesture as? UIPanGestureRecognizer else { return }
switch gesture.state {
case .began, .changed, .ended:
updateHeaderConstraint(gesture : panGesture)
default:
break
}
}
func updateHeaderConstraint(gesture pan : UIPanGestureRecognizer) {
// if gesture is starting, keep track of the start constraint value
if pan.state == .began {
headerViewTopConstraintStartValue = headerViewTopConstraint.constant
}
if pan.state == .began || pan.state == .changed {
// use the correct view for the given gesture's translate
let translation = pan.translation(in: pan is UIScreenEdgePanGestureRecognizer ? view : headerView)
// add the translation to the start value
headerViewTopConstraint.constant = headerViewTopConstraintStartValue + translation.y
// if the constant is now over 1, we will stretch downward. We use a sine easeOut to give
// a rubber-band effect
if headerViewTopConstraint.constant > 0 {
let t = Float(headerViewTopConstraint.constant) / Float(UIScreen.main.bounds.height)
let t2 = CGFloat(Curve.sine.easeOut(t))
//og.info("\(headerViewTopConstraint.constant) -> \(t) -> \(t2)")
headerViewTopConstraint.constant = t2 * headerView.frame.height * 2.0
}
} else if pan.state == .ended {
// gesture is ending, we will animate to either visible or hidden
let newTopY : CGFloat = headerViewTopConstraint.constant > -(headerView.frame.height * 0.25) ? 0.0 : (-self.headerView.frame.height - 1.0)
// we also need a display link here to properly communicate the new headerY from the presentation layer to
// the metal renderer.
self.displayLink = CADisplayLink(target: self, selector: #selector(displayLinkDidFire))
self.displayLink!.add(to: .main, forMode: .common)
UIView.animate(withDuration: 0.2) { [weak self] in
// animate to the new position
self?.headerViewTopConstraint.constant = newTopY
self?.view.layoutIfNeeded()
} completion: { [weak self] _ in
// all done, we kill the display link
self?.displayLink?.invalidate()
self?.displayLink = nil
}
}
}
@objc func displayLinkDidFire(_ displayLink: CADisplayLink) {
// save the actual animating value of the header view's Y position
self.headerViewY = self.headerView.layer.presentation()?.frame.minY ?? 0
}
func sendControllerThumbstickUpdate() {
guard let sc = streamingConnection else { return }
guard let gc = gameController else { return }
let snapshot = gc.capture()
guard let gp = snapshot.extendedGamepad else { return }
guard let controller = gamepads[gc.playerIndex] else { return }
// Fallback to default funcationality if we haven't received an ID from UE. This could happen if using latest app with UE < 5.2
let controllerIndex = controller.id ?? UInt8(gc.playerIndex.rawValue)
// dict of type -> oldValue, newValue
let inputs : [StreamingConnectionControllerInputType : ( oldValue: Float?, newValue : Float)] = [
.thumbstickLeftX : ( self.gameControllerSnapshot?.extendedGamepad?.leftThumbstick.xAxis.value, gp.leftThumbstick.xAxis.value),
.thumbstickLeftY : ( self.gameControllerSnapshot?.extendedGamepad?.leftThumbstick.yAxis.value, gp.leftThumbstick.yAxis.value),
.thumbstickRightX : ( self.gameControllerSnapshot?.extendedGamepad?.rightThumbstick.xAxis.value, gp.rightThumbstick.xAxis.value),
.thumbstickRightY : ( self.gameControllerSnapshot?.extendedGamepad?.rightThumbstick.yAxis.value, gp.rightThumbstick.yAxis.value),
]
for input in inputs {
if (input.value.newValue != 0.0) || ((input.value.oldValue ?? 0.0) != input.value.newValue) {
sc.sendControllerAnalog(input.key, controllerIndex: UInt8(controllerIndex), value: input.value.newValue)
}
}
self.gameControllerSnapshot = snapshot
}
func buttonValueChanged(button : GCControllerButtonInput, value : Float, pressed : Bool) {
guard let sc = streamingConnection else { return }
guard let gc = gameController else { return }
guard let gp = gc.extendedGamepad else { return }
guard let controller = gamepads[gc.playerIndex] else { return }
// Fallback to default functionality if we haven't received an ID from UE. This could happen if using latest app with UE < 5.2
let controllerIndex = controller.id ?? UInt8(gc.playerIndex.rawValue)
let isRepeat = false
var inputType : StreamingConnectionControllerInputType!
switch button {
case gp.leftThumbstickButton:
inputType = .thumbstickLeftButton
case gp.rightThumbstickButton:
inputType = .thumbstickRightButton
case gp.buttonA:
inputType = .faceButtonBottom
case gp.buttonB:
inputType = .faceButtonRight
case gp.buttonX:
inputType = .faceButtonLeft
case gp.buttonY:
inputType = .faceButtonTop
case gp.leftShoulder:
inputType = .shoulderButtonLeft
case gp.rightShoulder:
inputType = .shoulderButtonRight
case gp.leftTrigger:
inputType = .triggerButtonLeft
case gp.rightTrigger:
inputType = .triggerButtonRight
case gp.dpad.up:
inputType = .dpadUp
case gp.dpad.down:
inputType = .dpadDown
case gp.dpad.left:
inputType = .dpadLeft
case gp.dpad.right:
inputType = .dpadRight
case gp.buttonOptions:
inputType = .specialButtonLeft
case gp.buttonMenu:
inputType = .specialButtonRight
default:
Log.warning("Couldn't find mapping for input button \(button.description)")
return
}
if button.isPressed {
sc.sendControllerButtonPressed(inputType!, controllerIndex: UInt8(controllerIndex), isRepeat: isRepeat)
} else {
sc.sendControllerButtonReleased(inputType!, controllerIndex: UInt8(controllerIndex))
}
}
func controllerConnected(gamepad: GCController) {
guard let sc = streamingConnection else { return }
sc.sendControllerConnected();
let newGamepad = Gamepad()
newGamepad.controller = gamepad
gamepads[gamepad.playerIndex] = newGamepad
}
func controllerDisconnected(gamepad: GCController) {
guard let sc = streamingConnection else { return }
sc.sendControllerDisconnected(controllerIndex: gamepads[gamepad.playerIndex]!.id!);
gamepads.removeValue(forKey: gamepad.playerIndex)
}
func controllerResponseReceived(controllerIndex: UInt8) {
for gamepad in gamepads.values {
if(gamepad.id == nil) {
gamepad.id = controllerIndex;
break;
}
}
}
}
extension VideoViewController : HeaderViewDelegate {
func headerViewStatsButtonTapped(_ headerView: HeaderView) {
self.showStats = !self.showStats
streamingConnection?.showStats(self.showStats)
}
func headerViewExitButtonTapped(_ headerView : HeaderView) {
let disconnectAlert = UIAlertController(title: nil, message: NSLocalizedString("Disconnect from the remote session?", comment: "Prompt disconnect from a remote session."), preferredStyle: .alert)
disconnectAlert.addAction(UIAlertAction(title: NSLocalizedString("Disconnect", comment: "Button to disconnect from a UE instance"), style: .destructive, handler: { [weak self] _ in
self?.exit()
}))
disconnectAlert.addAction(UIAlertAction(title: Localized.buttonCancel(), style: .cancel))
self.present(disconnectAlert, animated:true)
}
func headerViewLogButtonTapped(_ headerView : HeaderView) {
performSegue(withIdentifier: "showLog", sender: headerView)
}
}
extension VideoViewController: UIPickerViewDelegate, UIPickerViewDataSource {
func numberOfComponents(in pickerView: UIPickerView) -> Int {
// Number of columns
return 1
}
func pickerView(_ pickerView: UIPickerView, numberOfRowsInComponent component: Int) -> Int {
// Number of rows
return pickerData.count
}
func pickerView(_ pickerView: UIPickerView, titleForRow row: Int, forComponent component: Int) -> String? {
return pickerData[row]
}
func pickerView(_ pickerView: UIPickerView, didSelectRow row: Int, inComponent component: Int) {
selectedStreamer = pickerData[row]
}
}