// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using EpicGames.Core;
using EpicGames.UHT.Utils;
namespace EpicGames.UHT.Tokenizer
{
///
/// Options for GetRawString method
///
[Flags]
public enum UhtRawStringOptions
{
///
/// No options
///
None,
///
/// Don't consider the terminator while in a quoted string
///
RespectQuotes = 1 << 0,
///
/// Don't consume the terminator. It will be parsed later.
///
DontConsumeTerminator = 1 << 1,
}
///
/// Helper methods for testing flags. These methods perform better than the generic HasFlag which hits
/// the GC and stalls.
///
public static class UhtRawStringOptionsExtensions
{
///
/// Test to see if any of the specified flags are set
///
/// Current flags
/// Flags to test for
/// True if any of the flags are set
public static bool HasAnyFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags)
{
return (inFlags & testFlags) != 0;
}
///
/// Test to see if all of the specified flags are set
///
/// Current flags
/// Flags to test for
/// True if all the flags are set
public static bool HasAllFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags)
{
return (inFlags & testFlags) == testFlags;
}
///
/// Test to see if a specific set of flags have a specific value.
///
/// Current flags
/// Flags to test for
/// Expected value of the tested flags
/// True if the given flags have a specific value.
public static bool HasExactFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags, UhtRawStringOptions matchFlags)
{
return (inFlags & testFlags) == matchFlags;
}
}
///
/// Interface invoked when the parser reaches a '#' preprocessor block
///
public interface IUhtTokenPreprocessor
{
///
/// Parse a preprocessor directive
///
/// Token starting the directive. Will be only the '#'
/// If true, the directive the source is being included. Otherwise it is being skipped as part of an #if block
/// If true, comments should be cleared
/// If true, excluded contents should be checked for unparsed UE macros (i.e. UCLASS)
/// True if the source should continue to be included
public bool ParsePreprocessorDirective(ref UhtToken token, bool isBeingIncluded, out bool clearComments, out bool illegalContentsCheck);
///
/// Save the current preprocessor state
///
public void SaveState();
///
/// Restore the current preprocessor state
///
public void RestoreState();
}
///
/// Common token reader interfaces for all token reader. When creating extension methods, use the interface.
///
public interface IUhtTokenReader : IUhtMessageSite
{
///
/// True if the reader is at the end of the stream
///
public bool IsEOF { get; }
///
/// Current input position in the stream by characters
///
public int InputPos { get; }
///
/// Current input line in the stream
///
public int InputLine { get; set; }
///
/// Preprocessor attached to the token reader
///
public IUhtTokenPreprocessor? TokenPreprocessor { get; set; }
///
/// If the reader doesn't have a current token, then read the next token and return a reference to it.
/// Otherwise return a reference to the current token.
///
/// The current token. Will be invalidated by other calls to ITokenReader
public ref UhtToken PeekToken();
///
/// Mark the current token as being consumed. Any call to PeekToken or GetToken will read another token.
///
public void ConsumeToken();
///
/// Get the next token in the data. If there is a current token, then that token is returned and marked as consumed.
///
///
public UhtToken GetToken();
///
/// Method only implemented by the replay reader that returns the current token index
///
/// Current token index
public int GetReplayTokenIndex();
///
/// Tests to see if the given token is the first token of a line
///
/// The token to test
/// True if the token is the first token on the line
public bool IsFirstTokenInLine(ref UhtToken token);
///
/// Skip any whitespace and comments at the current buffer position
///
public void SkipWhitespaceAndComments();
///
/// Read the entire next line in the buffer
///
///
public UhtToken GetLine();
///
/// Get a view of the buffer being read
///
/// Starting character offset in the buffer.
/// Length of the span
/// The string view into the buffer
public StringView GetStringView(int startPos, int count);
///
/// Return a string terminated by the given character.
///
/// The character to stop at.
/// Options
/// The parsed string
public StringView GetRawString(char terminator, UhtRawStringOptions options);
///
/// The current collection of parsed comments. This does not include any comments parsed as part of a
/// call to PeekToken unless ConsumeToken has been invoked after a call to PeekToken.
///
public ReadOnlySpan Comments { get; }
///
/// Clear the current collection of comments. Any comments parsed by PeekToken prior to calling ConsomeToken will
/// not be cleared.
///
public void ClearComments();
///
/// Disable the processing of comments. This is often done when skipping a bulk of the buffer.
///
///
public void DisableComments();
///
/// Enable comment collection.
///
public void EnableComments();
///
/// If there are any pending comments (due to a PeekToken), commit then so they will be return as current comments
///
//COMPATIBILITY-TODO - Remove once the struct adding of tooltips is fixed
public void CommitPendingComments();
///
/// Save the current parsing state. There is a limited number of states that can be saved.
/// Invoke either RestoreState or AbandonState after calling SaveState.
///
public void SaveState();
///
/// Restore a previously saved state.
///
public void RestoreState();
///
/// Abandon a previously saved state
///
public void AbandonState();
///
/// Enable the recording of tokens
///
public void EnableRecording();
///
/// Disable the recording of tokens. Any currently recorded tokens will be removed
///
public void DisableRecording();
///
/// Record the given token to the list of recorded tokens
///
/// Token to record
public void RecordToken(ref UhtToken token);
///
/// Get the current collection of recorded tokens
///
public List RecordedTokens { get; }
}
///
/// Represents a list of tokens. Follow the Next chain for each element in the list.
///
public class UhtTokenList
{
///
/// The token
///
public UhtToken Token { get; set; }
///
/// The next token in the list
///
public UhtTokenList? Next { get; set; }
///
/// Join the tokens in the list
///
/// Destination builder
/// Separator between the tokens
///
public void Join(StringBuilder builder, string separator = "")
{
builder.Append(Token.Value.ToString());
UhtTokenList list = this;
while (list.Next != null)
{
list = list.Next;
builder.Append(separator);
builder.Append(list.Token.Value.ToString());
}
}
///
/// Join the tokens in the list
///
/// Separator between the tokens
///
public string Join(string separator = "")
{
if (Next == null)
{
return Token.Value.ToString();
}
StringBuilder builder = new();
Join(builder, separator);
return builder.ToString();
}
///
/// Return the token list as an array
///
///
public UhtToken[] ToArray()
{
int count = 1;
for (UhtTokenList temp = this; temp.Next != null; temp = temp.Next)
{
++count;
}
UhtToken[] outTokens = new UhtToken[count];
outTokens[0] = Token;
count = 1;
for (UhtTokenList temp = this; temp.Next != null; temp = temp.Next)
{
outTokens[count] = temp.Next.Token;
++count;
}
return outTokens;
}
}
///
/// Token list cache. Token lists must be returned to the cache.
///
public static class UhtTokenListCache
{
private static readonly ThreadLocal s_tls = new(() => null);
///
/// Borrow a token list
///
/// Starting token
/// Token list
public static UhtTokenList Borrow(UhtToken token)
{
UhtTokenList? identifier = s_tls.Value;
if (identifier != null)
{
s_tls.Value = identifier.Next;
}
else
{
identifier = new UhtTokenList();
}
identifier.Token = token;
identifier.Next = null;
return identifier;
}
///
/// Return a token list to the cache
///
///
public static void Return(UhtTokenList identifier)
{
UhtTokenList? tail = s_tls.Value;
if (tail != null)
{
tail.Next = identifier;
}
for (; identifier.Next != null; identifier = identifier.Next)
{
}
s_tls.Value = identifier;
}
}
///
/// Delegate for when a token is parsed
///
/// The token in question
public delegate void UhtTokenDelegate(ref UhtToken token);
///
/// Delegate for when a token is parsed in an until block
///
/// The token in question
/// True if parsing should continue
public delegate bool UhtTokensUntilDelegate(ref UhtToken token);
///
/// Delegate for an enumeration of tokens
///
/// Parsed tokens
public delegate void UhtTokensDelegate(IEnumerable tokens);
///
/// Delegate for cached token list
///
/// Token list that can be cached
public delegate void UhtTokenListDelegate(UhtTokenList tokenList);
///
/// Delegate for a constant float
///
/// Value in question
public delegate void UhtTokenConstFloatDelegate(float value);
///
/// Delegate for a constant double
///
/// Value in question
public delegate void UhtTokenConstDoubleDelegate(double value);
///
/// Helper struct to disable comment parsing. Should be used in a using block
///
public readonly struct UhtTokenDisableComments : IDisposable
{
private readonly IUhtTokenReader _tokenReader;
///
/// Construct instance
///
/// Token reader to disable
public UhtTokenDisableComments(IUhtTokenReader tokenReader)
{
_tokenReader = tokenReader;
_tokenReader.DisableComments();
}
///
/// Enable comments
///
public void Dispose()
{
_tokenReader.EnableComments();
}
}
///
/// Helper struct to save token reader state
///
public struct UhtTokenSaveState : IDisposable
{
private readonly IUhtTokenReader _tokenReader;
private bool _handled;
///
/// Construct instance
///
/// Token reader
public UhtTokenSaveState(IUhtTokenReader tokenReader)
{
_tokenReader = tokenReader;
_handled = false;
_tokenReader.SaveState();
}
///
/// Restore the token reader state
///
public void Dispose()
{
if (!_handled)
{
RestoreState();
}
}
///
/// Restore the token reader state
///
/// Thrown if state has already been restored or aborted
public void RestoreState()
{
if (_handled)
{
throw new UhtIceException("Can not call RestoreState/AbandonState more than once");
}
_tokenReader.RestoreState();
_handled = true;
}
///
/// Abandon the saved state
///
/// Thrown if state has already been restored or aborted
public void AbandonState()
{
if (_handled)
{
throw new UhtIceException("Can not call RestoreState/AbandonState more than once");
}
_tokenReader.AbandonState();
_handled = true;
}
}
}