// Copyright Epic Games, Inc. All Rights Reserved. #include "QuicClient.h" void FQuicClient::StartEndpoint() { QUIC_STATUS Status; /** * Allocate new connection object */ if (QUIC_FAILED(Status = MsQuic->ConnectionOpen(Registration, ClientConnectionCallback, this, &ClientConnection))) { ConnectionState = EQuicClientState::Failed; EndpointError = QuicUtils::ConvertQuicStatus(Status); UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not open connection: %s."), *QuicUtils::GetEndpointErrorString(EndpointError)); return; } if (ClientConfig->EncryptionMode == EEncryptionMode::Disabled) { constexpr BOOLEAN Value = true; MsQuic->SetParam(ClientConnection, QUIC_PARAM_CONN_DISABLE_1RTT_ENCRYPTION, sizeof(Value), &Value); } /** * Start connection to the server */ if (QUIC_FAILED(Status = MsQuic->ConnectionStart(ClientConnection, QuicConfig, QUIC_ADDRESS_FAMILY_INET, StringCast(*Endpoint.Address.ToString()).Get(), Endpoint.Port))) { ConnectionState = EQuicClientState::Failed; EndpointError = QuicUtils::ConvertQuicStatus(Status); UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not start connection: %s."), *QuicUtils::GetEndpointErrorString(EndpointError)); return; } UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Started client.")); } void FQuicClient::StopEndpoint() { if (IsStopping()) { return; } UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Stopping client ...")); Stop(); if (ConnectionState == EQuicClientState::Connecting) { ConnectionState = EQuicClientState::Disconnected; } if (MsQuic) { if (ClientConnection) { // Only do this if the connection wasn't already shutdown/closed by the peer or transport if (ConnectionState != EQuicClientState::Stopped) { if ((EndpointError == EQuicEndpointError::Normal || EndpointError == EQuicEndpointError::ConnectionAbort) && ConnectionState != EQuicClientState::Closed) { MsQuic->ConnectionShutdown(ClientConnection, QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); MsQuic->ConnectionClose(ClientConnection); } ConnectionState = EQuicClientState::Stopped; } } } UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Stopped client.")); } bool FQuicClient::IsEndpointReady() { return ConnectionState == EQuicClientState::Connected; } void FQuicClient::UpdateEndpoint() { if (IsStopping() || ConnectionState != EQuicClientState::Connected) { return; } ConsumeMessages(); } void FQuicClient::ConsumeInboundMessages() { if (InboundMessages.IsEmpty()) { return; } double CurrentTime = FPlatformTime::Seconds(); const double ConsumeLimit = CurrentTime + MaxConsumeTime.GetTotalSeconds(); FInboundMessage InboundMessage; while (CurrentTime < ConsumeLimit && InboundMessages.Dequeue(InboundMessage)) { MessageInboundDelegate.ExecuteIfBound(InboundMessage); CurrentTime = FPlatformTime::Seconds(); } } void FQuicClient::ProcessInboundBuffers() { if (InboundQuicBuffers.IsEmpty()) { return; } double CurrentTime = FPlatformTime::Seconds(); const double ConsumeLimit = CurrentTime + MaxConsumeTime.GetTotalSeconds(); FInboundQuicBuffer InboundBuffer; while (CurrentTime < ConsumeLimit && InboundQuicBuffers.Dequeue(InboundBuffer)) { if (ReceiveFromStream(InboundBuffer)) { if (InboundBuffer.bEndOfStream) { ReachedEndOfStream(InboundBuffer); } } CurrentTime = FPlatformTime::Seconds(); } } void FQuicClient::ConsumeOutboundMessages() { if (OutboundMessages.IsEmpty()) { return; } double CurrentTime = FPlatformTime::Seconds(); const double ConsumeLimit = CurrentTime + MaxConsumeTime.GetTotalSeconds(); FOutboundMessage OutboundMessage; while (CurrentTime < ConsumeLimit && OutboundMessages.Dequeue(OutboundMessage)) { SendToStream(OutboundMessage); CurrentTime = FPlatformTime::Seconds(); } } void FQuicClient::CollectStatistics() { if (IsStopping() || ConnectionState != EQuicClientState::Connected) { return; } if (MsQuic == nullptr) { return; } QUIC_STATISTICS Stats{}; QUIC_STATUS Status; if (QUIC_FAILED(Status = GetConnectionStats(ClientConnection, Stats))) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not collect statistics: %s."), *QuicUtils::ConvertResult(Status)); return; } FMessageTransportStatistics TransportStats = ConvertStatistics(Endpoint, Stats); StatisticsUpdatedDelegate.ExecuteIfBound(TransportStats); } void FQuicClient::EnqueueOutboundMessage(const FOutboundMessage& OutboundMessage) { OutboundMessages.Enqueue(OutboundMessage); } void FQuicClient::SendHello(FIPv4Endpoint RemoteEndpoint) { if (IsStopping() || ConnectionState != EQuicClientState::Connected) { return; } FMessageHeader MessageHeader(EQuicMessageType::Hello, FGuid(), LocalNodeId, 0); const FQuicPayloadPtr MetaBuffer = SerializeHeaderDelegate.Execute(MessageHeader); const FOutboundMessage OutboundMessage(RemoteEndpoint, MetaBuffer); EnqueueOutboundMessage(OutboundMessage); UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Sent hello.")); } void FQuicClient::SendBye() { if (IsStopping() || ConnectionState != EQuicClientState::Connected) { return; } FMessageHeader MessageHeader(EQuicMessageType::Bye, FGuid(), LocalNodeId, 0); const FQuicPayloadPtr MetaBuffer = SerializeHeaderDelegate.Execute(MessageHeader); const FOutboundMessage OutboundMessage(Endpoint, MetaBuffer); SendToStream(OutboundMessage); UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Sent bye.")); } _Function_class_(QUIC_CONNECTION_CALLBACK) QUIC_STATUS FQuicClient::ClientConnectionCallback(HQUIC Connection, void* Context, QUIC_CONNECTION_EVENT* Event) { FQuicClient* ClientContext = static_cast(Context); switch (Event->Type) { case QUIC_CONNECTION_EVENT_CONNECTED: /** * Connection handshake has completed */ ClientContext->OnConnectionConnected(); break; case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT: /** * Idle timeout killed the connection */ ClientContext->OnConnectionShutdownByTransport(Event->SHUTDOWN_INITIATED_BY_TRANSPORT.Status); break; case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER: /** * Peer killed the connection */ ClientContext->OnConnectionShutdownByPeer(Event->SHUTDOWN_INITIATED_BY_PEER.ErrorCode); break; case QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE: /** * Connection shutdown complete, ready to cleanup */ ClientContext->OnConnectionShutdownComplete(Connection, Event->SHUTDOWN_COMPLETE.AppCloseInProgress); break; case QUIC_CONNECTION_EVENT_RESUMPTION_TICKET_RECEIVED: /** * Resumption ticket received. Does not do anything at this time */ ClientContext->OnConnectionResumptionTicketReceived(); break; case QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED: { /** * Peer started a stream, set local stream callback handler */ FIPv4Endpoint* PeerAddress = new FIPv4Endpoint(ClientContext->Endpoint); FStreamCallbackInfo* StreamCallbackInfo = new FStreamCallbackInfo(); StreamCallbackInfo->Endpoint = ClientContext; StreamCallbackInfo->PeerAddress = PeerAddress; ClientContext->MsQuic->SetCallbackHandler(Event->PEER_STREAM_STARTED.Stream, reinterpret_cast(ClientStreamCallback), StreamCallbackInfo); ClientContext->OnConnectionPeerStreamStarted(Event->PEER_STREAM_STARTED.Stream); break; } case QUIC_CONNECTION_EVENT_DATAGRAM_RECEIVED: case QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED: case QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED: case QUIC_CONNECTION_EVENT_IDEAL_PROCESSOR_CHANGED: case QUIC_CONNECTION_EVENT_LOCAL_ADDRESS_CHANGED: case QUIC_CONNECTION_EVENT_PEER_ADDRESS_CHANGED: case QUIC_CONNECTION_EVENT_PEER_CERTIFICATE_RECEIVED: case QUIC_CONNECTION_EVENT_PEER_NEEDS_STREAMS: case QUIC_CONNECTION_EVENT_RESUMED: case QUIC_CONNECTION_EVENT_STREAMS_AVAILABLE: break; default: UE_LOG(LogQuicMessagingTransport, Warning, TEXT("[QuicClient] Received unhandled connection event: %d"), Event->Type); break; } return QUIC_STATUS_SUCCESS; } _Function_class_(QUIC_STREAM_CALLBACK) QUIC_STATUS FQuicClient::ClientStreamCallback(HQUIC Stream, void* Context, QUIC_STREAM_EVENT* Event) { const FStreamCallbackInfo* CallbackInfo = static_cast(Context); FQuicClient* ClientContext = static_cast(CallbackInfo->Endpoint); switch (Event->Type) { case QUIC_STREAM_EVENT_SEND_COMPLETE: /** * Previous StreamSend has completed, context returned */ ClientContext->OnStreamSendComplete(Stream, Event); break; case QUIC_STREAM_EVENT_RECEIVE: /** * Data received from the peer */ ClientContext->OnStreamReceive(Stream, *CallbackInfo->PeerAddress, Event); return QUIC_STATUS_PENDING; case QUIC_STREAM_EVENT_PEER_SEND_ABORTED: /** * Peer aborted its stream send direction */ ClientContext->OnStreamPeerSendAborted(Stream); break; case QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN: /** * Peer gracefully shut down its stream send direction */ break; case QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE: /** * Both stream directions have been shut down, safe to cleanup */ ClientContext->OnStreamShutdownComplete(Stream); delete CallbackInfo; break; case QUIC_STREAM_EVENT_START_COMPLETE: case QUIC_STREAM_EVENT_PEER_ACCEPTED: case QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED: case QUIC_STREAM_EVENT_IDEAL_SEND_BUFFER_SIZE: case QUIC_STREAM_EVENT_SEND_SHUTDOWN_COMPLETE: break; default: UE_LOG(LogQuicMessagingTransport, Warning, TEXT("[QuicClient] Received unhandled stream event: %d"), Event->Type); break; } return QUIC_STATUS_SUCCESS; } void FQuicClient::OnConnectionConnected() { ConnectionState = EQuicClientState::Connected; EndpointConnectedDelegate.ExecuteIfBound(Endpoint); UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Handshake completed. Server: %s / LocalNodeId: %s / ThreadId: %d."), *Endpoint.ToString(), *LocalNodeId.ToString(), Thread->GetThreadID()); } void FQuicClient::OnConnectionShutdownByTransport(QuicUtils::HRESULT Status) { ConnectionState = EQuicClientState::Disconnected; bTransportClosed = true; EndpointError = QuicUtils::ConvertQuicStatus(Status); UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Connection end (by transport). Server: %s."), *Endpoint.ToString()); if (EndpointError != EQuicEndpointError::Normal) { UE_LOG(LogQuicMessagingTransport, Warning, TEXT("[QuicClient] Shutdown reason: %s"), *QuicUtils::GetEndpointErrorString(EndpointError)); } } void FQuicClient::OnConnectionShutdownByPeer(QUIC_UINT62 ErrorCode) { ConnectionState = EQuicClientState::Disconnected; bTransportClosed = true; UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Connection end (by peer). Server: %s / Error code: %d."), *Endpoint.ToString(), ErrorCode); } void FQuicClient::OnConnectionShutdownComplete(HQUIC Connection, BOOLEAN AppCloseInProgress) { if (!AppCloseInProgress) { UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Connection shutdown complete. Server: %s."), *Endpoint.ToString()); if (MsQuic && ConnectionState != EQuicClientState::Stopped) { if (ConnectionState != EQuicClientState::Closed) { MsQuic->ConnectionClose(Connection); ConnectionState = EQuicClientState::Closed; } EndpointDisconnectedDelegate.ExecuteIfBound(Endpoint, EndpointError); } } } void FQuicClient::OnConnectionResumptionTicketReceived() { UE_LOG(LogQuicMessagingTransport, Verbose, TEXT("[QuicClient] Resumption ticket received.")); } void FQuicClient::OnConnectionPeerStreamStarted(HQUIC Stream) { const uint64 StreamId = GetStreamId(Stream); { const FQuicPayloadPtr InboundData = MakeShared, ESPMode::ThreadSafe>(); FScopeLock InboundLock(&InboundBuffersCS); const QUIC_ADDR LocalAddress = GetAddressFromEndpoint(Endpoint); const FInboundBuffer InboundBuffer(LocalAddress, FIPv4Endpoint::Any, InboundData); const FAddrStream AddressStreamKey(Endpoint, StreamId); InboundBuffers.Add(AddressStreamKey, InboundBuffer); } UE_LOG(LogQuicMessagingTransport, VeryVerbose, TEXT("[QuicClient] Peer stream started. Server: %s / StreamId: %d."), *Endpoint.ToString(), StreamId); } void FQuicClient::OnStreamShutdownComplete(HQUIC Stream) const { if (MsQuic) { MsQuic->StreamClose(Stream); } } void FQuicClient::SendToStream(const FOutboundMessage& OutboundMessage) { HQUIC Stream = nullptr; QUIC_STATUS Status; FIPv4Endpoint* PeerAddress = new FIPv4Endpoint(Endpoint); FStreamCallbackInfo* StreamCallbackInfo = new FStreamCallbackInfo(); StreamCallbackInfo->Endpoint = this; StreamCallbackInfo->PeerAddress = PeerAddress; /** * Create/allocate a new unidirectional stream */ if (QUIC_FAILED(Status = MsQuic->StreamOpen(ClientConnection, QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL, ClientStreamCallback, StreamCallbackInfo, &Stream))) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not open stream: %s."), *QuicUtils::ConvertResult(Status)); return; } /** * Start the stream */ if (QUIC_FAILED(Status = MsQuic->StreamStart(Stream, QUIC_STREAM_START_FLAG_NONE))) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not start stream: %s."), *QuicUtils::ConvertResult(Status)); MsQuic->StreamClose(Stream); return; } const uint64 StreamId = GetStreamId(Stream); /** * Send buffers over the stream */ const FQuicBufferRef MetaBuffer = MakeShared(); const FQuicBufferRef MessageBuffer = MakeShared(); MetaBuffer->Buffer = OutboundMessage.SerializedHeader->GetData(); MetaBuffer->Length = OutboundMessage.SerializedHeader->Num(); RegisterStreamMessage(StreamId, *PeerAddress, !OutboundMessage.bHasPayload, OutboundMessage, MetaBuffer, MessageBuffer); if (QUIC_FAILED(Status = MsQuic->StreamSend(Stream, &*MetaBuffer, 1, OutboundMessage.bHasPayload ? QUIC_SEND_FLAG_NONE : QUIC_SEND_FLAG_FIN, StreamCallbackInfo))) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not send meta on stream: %s."), *QuicUtils::ConvertResult(Status)); MsQuic->StreamShutdown(Stream, QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 0); return; } // This message is header-only if (!OutboundMessage.bHasPayload) { return; } MessageBuffer->Buffer = OutboundMessage.SerializedPayload->GetData(); MessageBuffer->Length = OutboundMessage.SerializedPayload->Num(); if (QUIC_FAILED(Status = MsQuic->StreamSend(Stream, &*MessageBuffer, 1, QUIC_SEND_FLAG_FIN, StreamCallbackInfo))) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Could not send message on stream: %s."), *QuicUtils::ConvertResult(Status)); MsQuic->StreamShutdown(Stream, QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 0); } } bool FQuicClient::ReceiveFromStream(const FInboundQuicBuffer& QuicBuffer) { FScopeLock InboundLock(&InboundBuffersCS); const FAddrStream AddressStreamKey(QuicBuffer.PeerAddress, QuicBuffer.StreamId); FInboundBuffer* Buffer = InboundBuffers.Find(AddressStreamKey); if (!Buffer) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Invalid inbound buffer in ReceiveFromStream %d."), QuicBuffer.StreamId); return false; } const FQuicPayloadPtr& InboundData = Buffer->InboundData; for (const TPair InboundBuffer : QuicBuffer.Data) { Buffer->InboundData->Append(InboundBuffer.Key, InboundBuffer.Value); } // Retrieve message header if (!Buffer->bHeaderDeserialized && InboundData->Num() >= QUIC_MESSAGE_HEADER_SIZE) { const FQuicPayloadPtr HeaderData = MakeShared, ESPMode::ThreadSafe>(); HeaderData->Append(InboundData->GetData(), QUIC_MESSAGE_HEADER_SIZE); const FMessageHeader MessageHeader = DeserializeHeaderDelegate.Execute(HeaderData); Buffer->MessageHeader = MessageHeader; Buffer->bHeaderDeserialized = true; InboundData->RemoveAt(0, QUIC_MESSAGE_HEADER_SIZE); const uint32& SerializedMessageSize = MessageHeader.SerializedMessageSize; InboundData->Reserve(SerializedMessageSize); } InboundLock.Unlock(); if (MsQuic) { MsQuic->StreamReceiveComplete(QuicBuffer.Stream, QuicBuffer.BufferLength); } return true; } void FQuicClient::ReachedEndOfStream(const FInboundQuicBuffer& QuicBuffer) { FInboundBuffer InboundBuffer; { const FAddrStream AddressStreamKey(QuicBuffer.PeerAddress, QuicBuffer.StreamId); FScopeLock InboundLock(&InboundBuffersCS); if (!InboundBuffers.RemoveAndCopyValue(AddressStreamKey, InboundBuffer)) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Invalid inbound buffer in ReachedEndOfStream.")); return; } } const FQuicPayloadPtr& InboundData = InboundBuffer.InboundData; const FMessageHeader& MessageHeader = InboundBuffer.MessageHeader; // Transmitted message size smaller than expected if (InboundData->Num() < static_cast(MessageHeader.SerializedMessageSize)) { UE_LOG(LogQuicMessagingTransport, Error, TEXT("[QuicClient] Transmitted message size smaller than expected %d / %d / %d."), QuicBuffer.StreamId, InboundData->Num(), MessageHeader.SerializedMessageSize); return; } FInboundMessage InboundMessage(InboundData, MessageHeader, Endpoint, Endpoint); InboundMessages.Enqueue(InboundMessage); }