// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Text; using System.Threading; using EpicGames.Core; using EpicGames.UHT.Utils; namespace EpicGames.UHT.Tokenizer { /// /// Options for GetRawString method /// [Flags] public enum UhtRawStringOptions { /// /// No options /// None, /// /// Don't consider the terminator while in a quoted string /// RespectQuotes = 1 << 0, /// /// Don't consume the terminator. It will be parsed later. /// DontConsumeTerminator = 1 << 1, } /// /// Helper methods for testing flags. These methods perform better than the generic HasFlag which hits /// the GC and stalls. /// public static class UhtRawStringOptionsExtensions { /// /// Test to see if any of the specified flags are set /// /// Current flags /// Flags to test for /// True if any of the flags are set public static bool HasAnyFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags) { return (inFlags & testFlags) != 0; } /// /// Test to see if all of the specified flags are set /// /// Current flags /// Flags to test for /// True if all the flags are set public static bool HasAllFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags) { return (inFlags & testFlags) == testFlags; } /// /// Test to see if a specific set of flags have a specific value. /// /// Current flags /// Flags to test for /// Expected value of the tested flags /// True if the given flags have a specific value. public static bool HasExactFlags(this UhtRawStringOptions inFlags, UhtRawStringOptions testFlags, UhtRawStringOptions matchFlags) { return (inFlags & testFlags) == matchFlags; } } /// /// Interface invoked when the parser reaches a '#' preprocessor block /// public interface IUhtTokenPreprocessor { /// /// Parse a preprocessor directive /// /// Token starting the directive. Will be only the '#' /// If true, the directive the source is being included. Otherwise it is being skipped as part of an #if block /// If true, comments should be cleared /// If true, excluded contents should be checked for unparsed UE macros (i.e. UCLASS) /// True if the source should continue to be included public bool ParsePreprocessorDirective(ref UhtToken token, bool isBeingIncluded, out bool clearComments, out bool illegalContentsCheck); /// /// Save the current preprocessor state /// public void SaveState(); /// /// Restore the current preprocessor state /// public void RestoreState(); } /// /// Common token reader interfaces for all token reader. When creating extension methods, use the interface. /// public interface IUhtTokenReader : IUhtMessageSite { /// /// True if the reader is at the end of the stream /// public bool IsEOF { get; } /// /// Current input position in the stream by characters /// public int InputPos { get; } /// /// Current input line in the stream /// public int InputLine { get; set; } /// /// Preprocessor attached to the token reader /// public IUhtTokenPreprocessor? TokenPreprocessor { get; set; } /// /// If the reader doesn't have a current token, then read the next token and return a reference to it. /// Otherwise return a reference to the current token. /// /// The current token. Will be invalidated by other calls to ITokenReader public ref UhtToken PeekToken(); /// /// Mark the current token as being consumed. Any call to PeekToken or GetToken will read another token. /// public void ConsumeToken(); /// /// Get the next token in the data. If there is a current token, then that token is returned and marked as consumed. /// /// public UhtToken GetToken(); /// /// Method only implemented by the replay reader that returns the current token index /// /// Current token index public int GetReplayTokenIndex(); /// /// Tests to see if the given token is the first token of a line /// /// The token to test /// True if the token is the first token on the line public bool IsFirstTokenInLine(ref UhtToken token); /// /// Skip any whitespace and comments at the current buffer position /// public void SkipWhitespaceAndComments(); /// /// Read the entire next line in the buffer /// /// public UhtToken GetLine(); /// /// Get a view of the buffer being read /// /// Starting character offset in the buffer. /// Length of the span /// The string view into the buffer public StringView GetStringView(int startPos, int count); /// /// Return a string terminated by the given character. /// /// The character to stop at. /// Options /// The parsed string public StringView GetRawString(char terminator, UhtRawStringOptions options); /// /// The current collection of parsed comments. This does not include any comments parsed as part of a /// call to PeekToken unless ConsumeToken has been invoked after a call to PeekToken. /// public ReadOnlySpan Comments { get; } /// /// Clear the current collection of comments. Any comments parsed by PeekToken prior to calling ConsomeToken will /// not be cleared. /// public void ClearComments(); /// /// Disable the processing of comments. This is often done when skipping a bulk of the buffer. /// /// public void DisableComments(); /// /// Enable comment collection. /// public void EnableComments(); /// /// If there are any pending comments (due to a PeekToken), commit then so they will be return as current comments /// //COMPATIBILITY-TODO - Remove once the struct adding of tooltips is fixed public void CommitPendingComments(); /// /// Save the current parsing state. There is a limited number of states that can be saved. /// Invoke either RestoreState or AbandonState after calling SaveState. /// public void SaveState(); /// /// Restore a previously saved state. /// public void RestoreState(); /// /// Abandon a previously saved state /// public void AbandonState(); /// /// Enable the recording of tokens /// public void EnableRecording(); /// /// Disable the recording of tokens. Any currently recorded tokens will be removed /// public void DisableRecording(); /// /// Record the given token to the list of recorded tokens /// /// Token to record public void RecordToken(ref UhtToken token); /// /// Get the current collection of recorded tokens /// public List RecordedTokens { get; } } /// /// Represents a list of tokens. Follow the Next chain for each element in the list. /// public class UhtTokenList { /// /// The token /// public UhtToken Token { get; set; } /// /// The next token in the list /// public UhtTokenList? Next { get; set; } /// /// Join the tokens in the list /// /// Destination builder /// Separator between the tokens /// public void Join(StringBuilder builder, string separator = "") { builder.Append(Token.Value.ToString()); UhtTokenList list = this; while (list.Next != null) { list = list.Next; builder.Append(separator); builder.Append(list.Token.Value.ToString()); } } /// /// Join the tokens in the list /// /// Separator between the tokens /// public string Join(string separator = "") { if (Next == null) { return Token.Value.ToString(); } StringBuilder builder = new(); Join(builder, separator); return builder.ToString(); } /// /// Return the token list as an array /// /// public UhtToken[] ToArray() { int count = 1; for (UhtTokenList temp = this; temp.Next != null; temp = temp.Next) { ++count; } UhtToken[] outTokens = new UhtToken[count]; outTokens[0] = Token; count = 1; for (UhtTokenList temp = this; temp.Next != null; temp = temp.Next) { outTokens[count] = temp.Next.Token; ++count; } return outTokens; } } /// /// Token list cache. Token lists must be returned to the cache. /// public static class UhtTokenListCache { private static readonly ThreadLocal s_tls = new(() => null); /// /// Borrow a token list /// /// Starting token /// Token list public static UhtTokenList Borrow(UhtToken token) { UhtTokenList? identifier = s_tls.Value; if (identifier != null) { s_tls.Value = identifier.Next; } else { identifier = new UhtTokenList(); } identifier.Token = token; identifier.Next = null; return identifier; } /// /// Return a token list to the cache /// /// public static void Return(UhtTokenList identifier) { UhtTokenList? tail = s_tls.Value; if (tail != null) { tail.Next = identifier; } for (; identifier.Next != null; identifier = identifier.Next) { } s_tls.Value = identifier; } } /// /// Delegate for when a token is parsed /// /// The token in question public delegate void UhtTokenDelegate(ref UhtToken token); /// /// Delegate for when a token is parsed in an until block /// /// The token in question /// True if parsing should continue public delegate bool UhtTokensUntilDelegate(ref UhtToken token); /// /// Delegate for an enumeration of tokens /// /// Parsed tokens public delegate void UhtTokensDelegate(IEnumerable tokens); /// /// Delegate for cached token list /// /// Token list that can be cached public delegate void UhtTokenListDelegate(UhtTokenList tokenList); /// /// Delegate for a constant float /// /// Value in question public delegate void UhtTokenConstFloatDelegate(float value); /// /// Delegate for a constant double /// /// Value in question public delegate void UhtTokenConstDoubleDelegate(double value); /// /// Helper struct to disable comment parsing. Should be used in a using block /// public readonly struct UhtTokenDisableComments : IDisposable { private readonly IUhtTokenReader _tokenReader; /// /// Construct instance /// /// Token reader to disable public UhtTokenDisableComments(IUhtTokenReader tokenReader) { _tokenReader = tokenReader; _tokenReader.DisableComments(); } /// /// Enable comments /// public void Dispose() { _tokenReader.EnableComments(); } } /// /// Helper struct to save token reader state /// public struct UhtTokenSaveState : IDisposable { private readonly IUhtTokenReader _tokenReader; private bool _handled; /// /// Construct instance /// /// Token reader public UhtTokenSaveState(IUhtTokenReader tokenReader) { _tokenReader = tokenReader; _handled = false; _tokenReader.SaveState(); } /// /// Restore the token reader state /// public void Dispose() { if (!_handled) { RestoreState(); } } /// /// Restore the token reader state /// /// Thrown if state has already been restored or aborted public void RestoreState() { if (_handled) { throw new UhtIceException("Can not call RestoreState/AbandonState more than once"); } _tokenReader.RestoreState(); _handled = true; } /// /// Abandon the saved state /// /// Thrown if state has already been restored or aborted public void AbandonState() { if (_handled) { throw new UhtIceException("Can not call RestoreState/AbandonState more than once"); } _tokenReader.AbandonState(); _handled = true; } } }