1049 lines
25 KiB
C#
1049 lines
25 KiB
C#
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using EpicGames.Core;
|
|
using EpicGames.UHT.Utils;
|
|
|
|
namespace EpicGames.UHT.Tokenizer
|
|
{
|
|
|
|
/// <summary>
|
|
/// Token reader for source buffers
|
|
/// </summary>
|
|
public sealed class UhtTokenBufferReader : IUhtTokenReader, IUhtMessageLineNumber
|
|
{
|
|
private static readonly StringView[] s_emptyComments = Array.Empty<StringView>();
|
|
|
|
private readonly IUhtMessageSite _messageSite;
|
|
private List<StringView>? _comments = null;
|
|
private readonly List<UhtToken> _recordedTokens = new();
|
|
private IUhtTokenPreprocessor? _tokenPreprocessor = null;
|
|
private UhtToken _currentToken = new(); // PeekToken must have been invoked first
|
|
private readonly ReadOnlyMemory<char> _data;
|
|
private int _prevPos = 0;
|
|
private int _prevLine = 1;
|
|
private bool _hasToken = false;
|
|
private int _preCurrentTokenInputPos = 0;
|
|
private int _preCurrentTokenInputLine = 1;
|
|
private int _inputPos = 0;
|
|
private int _inputLine = 1;
|
|
private int _commentsDisableCount = 0;
|
|
private int _committedComments = 0;
|
|
private List<StringView>? _savedComments = null;
|
|
private int _savedInputPos = 0;
|
|
private int _savedInputLine = 0;
|
|
private bool _hasSavedState = false;
|
|
private bool _recordTokens = false;
|
|
private int _preprocessorPendingCommentsCount = 0;
|
|
|
|
/// <summary>
|
|
/// Construct a new token reader
|
|
/// </summary>
|
|
/// <param name="messageSite">Message site for messages</param>
|
|
/// <param name="input">Input source</param>
|
|
public UhtTokenBufferReader(IUhtMessageSite messageSite, ReadOnlyMemory<char> input)
|
|
{
|
|
_messageSite = messageSite;
|
|
_data = input;
|
|
}
|
|
|
|
#region IUHTMessageSite Implementation
|
|
IUhtMessageSession IUhtMessageSite.MessageSession => _messageSite.MessageSession;
|
|
IUhtMessageSource? IUhtMessageSite.MessageSource => _messageSite.MessageSource;
|
|
IUhtMessageLineNumber? IUhtMessageSite.MessageLineNumber => this;
|
|
#endregion
|
|
|
|
#region IUHTMessageLinenumber Implementation
|
|
int IUhtMessageLineNumber.MessageLineNumber => InputLine;
|
|
#endregion
|
|
|
|
#region ITokenReader Implementation
|
|
/// <inheritdoc/>
|
|
public bool IsEOF
|
|
{
|
|
get
|
|
{
|
|
if (_hasToken)
|
|
{
|
|
return _currentToken.TokenType.IsEndType();
|
|
}
|
|
else
|
|
{
|
|
return InputPos == _data.Length;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public int InputPos => _hasToken ? _preCurrentTokenInputPos : _inputPos;
|
|
|
|
/// <inheritdoc/>
|
|
public int InputLine
|
|
{
|
|
get => _hasToken ? _preCurrentTokenInputLine : _inputLine;
|
|
set
|
|
{
|
|
ClearToken();
|
|
_inputLine = value;
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public ReadOnlySpan<StringView> Comments
|
|
{
|
|
get
|
|
{
|
|
if (_comments != null && _committedComments != 0)
|
|
{
|
|
return new ReadOnlySpan<StringView>(_comments.ToArray(), 0, _committedComments);
|
|
}
|
|
else
|
|
{
|
|
return new ReadOnlySpan<StringView>(UhtTokenBufferReader.s_emptyComments);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public IUhtTokenPreprocessor? TokenPreprocessor { get => _tokenPreprocessor; set => _tokenPreprocessor = value; }
|
|
|
|
/// <inheritdoc/>
|
|
public ref UhtToken PeekToken()
|
|
{
|
|
if (!_hasToken)
|
|
{
|
|
_currentToken = GetTokenInternal(true);
|
|
_hasToken = true;
|
|
}
|
|
return ref _currentToken;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void SkipWhitespaceAndComments()
|
|
{
|
|
if (!_hasToken)
|
|
{
|
|
bool gotInlineComment = false;
|
|
SkipWhitespaceAndCommentsInternal(ref gotInlineComment, true);
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void ConsumeToken()
|
|
{
|
|
if (_recordTokens && !_currentToken.IsEndType())
|
|
{
|
|
_recordedTokens.Add(_currentToken);
|
|
}
|
|
_hasToken = false;
|
|
|
|
// When comments are disabled, we are still collecting comments, but aren't committing them
|
|
if (_commentsDisableCount == 0)
|
|
{
|
|
if (_comments != null)
|
|
{
|
|
_committedComments = _comments.Count;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ClearPendingComments();
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public UhtToken GetToken()
|
|
{
|
|
UhtToken token = PeekToken();
|
|
ConsumeToken();
|
|
return token;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public int GetReplayTokenIndex()
|
|
{
|
|
throw new NotImplementedException();
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public StringView GetRawString(char terminator, UhtRawStringOptions options)
|
|
{
|
|
ReadOnlySpan<char> span = _data.Span;
|
|
|
|
ClearToken();
|
|
SkipWhitespaceAndComments();
|
|
|
|
int startPos = InputPos;
|
|
bool inQuotes = false;
|
|
while (true)
|
|
{
|
|
char c = InternalGetChar(span);
|
|
|
|
// Check for end of file
|
|
if (c == 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Check for end of line
|
|
if (c == '\r' || c == '\n')
|
|
{
|
|
--_inputPos;
|
|
break;
|
|
}
|
|
|
|
// Check for terminator as long as we aren't in quotes
|
|
if (c == terminator && !inQuotes)
|
|
{
|
|
if (options.HasAnyFlags(UhtRawStringOptions.DontConsumeTerminator))
|
|
{
|
|
--_inputPos;
|
|
}
|
|
break;
|
|
}
|
|
|
|
// Check for comment
|
|
if (!inQuotes && c == '/')
|
|
{
|
|
char p = InternalPeekChar(span);
|
|
if (p == '*' || p == '/')
|
|
{
|
|
--_inputPos;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Check for quotes
|
|
if (c == '"' && options.HasAnyFlags(UhtRawStringOptions.RespectQuotes))
|
|
{
|
|
inQuotes = !inQuotes;
|
|
}
|
|
}
|
|
|
|
// If EOF, then error
|
|
if (inQuotes)
|
|
{
|
|
throw new UhtException(this, "Unterminated quoted string");
|
|
}
|
|
|
|
// Remove trailing whitespace
|
|
int endPos = InputPos;
|
|
for (; endPos > startPos; --endPos)
|
|
{
|
|
char c = span[endPos - 1];
|
|
if (c != ' ' && c != '\t')
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Check for too long
|
|
if (endPos - startPos >= UhtToken.MaxStringLength)
|
|
{
|
|
throw new UhtException(this, $"String exceeds maximum of {UhtToken.MaxStringLength} characters");
|
|
}
|
|
return new StringView(_data, startPos, endPos - startPos);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public UhtToken GetLine()
|
|
{
|
|
ReadOnlySpan<char> span = _data.Span;
|
|
|
|
ClearToken();
|
|
_prevPos = _inputPos;
|
|
_prevLine = _inputLine;
|
|
|
|
if (_prevPos == span.Length)
|
|
{
|
|
return new UhtToken(UhtTokenType.EndOfFile, _prevPos, _prevLine, _prevPos, _prevLine, new StringView(_data.Slice(_prevPos, 0)));
|
|
}
|
|
|
|
int lastPos = _inputPos;
|
|
while (true)
|
|
{
|
|
char c = InternalGetChar(span);
|
|
if (c == 0)
|
|
{
|
|
break;
|
|
}
|
|
else if (c == '\r')
|
|
{
|
|
}
|
|
else if (c == '\n')
|
|
{
|
|
++_inputLine;
|
|
break;
|
|
}
|
|
else
|
|
{
|
|
lastPos = _inputPos;
|
|
}
|
|
}
|
|
|
|
return new UhtToken(UhtTokenType.Line, _prevPos, _prevLine, _prevPos, _prevLine, new StringView(_data[_prevPos..lastPos]));
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public StringView GetStringView(int startPos, int count)
|
|
{
|
|
return new StringView(_data, startPos, count);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void ClearComments()
|
|
{
|
|
if (_comments != null)
|
|
{
|
|
|
|
// Clearing comments does not remove any uncommitted comments
|
|
_comments.RemoveRange(0, _committedComments);
|
|
_committedComments = 0;
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void DisableComments()
|
|
{
|
|
++_commentsDisableCount;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void EnableComments()
|
|
{
|
|
--_commentsDisableCount;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void CommitPendingComments()
|
|
{
|
|
if (_comments != null)
|
|
{
|
|
_committedComments = _comments.Count;
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public bool IsFirstTokenInLine(ref UhtToken token)
|
|
{
|
|
return IsFirstTokenInLine(_data.Span, token.InputStartPos);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void SaveState()
|
|
{
|
|
if (_hasSavedState)
|
|
{
|
|
throw new UhtIceException("Can not save more than one state");
|
|
}
|
|
_hasSavedState = true;
|
|
_savedInputLine = InputLine;
|
|
_savedInputPos = InputPos;
|
|
if (_comments != null)
|
|
{
|
|
_savedComments ??= new List<StringView>();
|
|
_savedComments.Clear();
|
|
_savedComments.AddRange(_comments);
|
|
}
|
|
_tokenPreprocessor?.SaveState();
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void RestoreState()
|
|
{
|
|
if (!_hasSavedState)
|
|
{
|
|
throw new UhtIceException("Can not restore state when none have been saved");
|
|
}
|
|
_hasSavedState = false;
|
|
ClearToken();
|
|
_inputPos = _savedInputPos;
|
|
_inputLine = _savedInputLine;
|
|
if (_savedComments != null && _comments != null)
|
|
{
|
|
_comments.Clear();
|
|
_comments.AddRange(_savedComments);
|
|
}
|
|
_tokenPreprocessor?.RestoreState();
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void AbandonState()
|
|
{
|
|
if (!_hasSavedState)
|
|
{
|
|
throw new UhtIceException("Can not abandon state when none have been saved");
|
|
}
|
|
_hasSavedState = false;
|
|
ClearToken();
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void EnableRecording()
|
|
{
|
|
if (_recordTokens)
|
|
{
|
|
throw new UhtIceException("Can not nest token recording");
|
|
}
|
|
_recordTokens = true;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void RecordToken(ref UhtToken token)
|
|
{
|
|
if (!_recordTokens)
|
|
{
|
|
throw new UhtIceException("Attempt to disable token recording when it isn't enabled");
|
|
}
|
|
_recordedTokens.Add(token);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public void DisableRecording()
|
|
{
|
|
if (!_recordTokens)
|
|
{
|
|
throw new UhtIceException("Attempt to disable token recording when it isn't enabled");
|
|
}
|
|
_recordedTokens.Clear();
|
|
_recordTokens = false;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public List<UhtToken> RecordedTokens
|
|
{
|
|
get
|
|
{
|
|
if (!_recordTokens)
|
|
{
|
|
throw new UhtIceException("Attempt to get recorded tokens when it isn't enabled");
|
|
}
|
|
return _recordedTokens;
|
|
}
|
|
}
|
|
#endregion
|
|
|
|
#region Internals
|
|
private void ClearAllComments()
|
|
{
|
|
if (_comments != null)
|
|
{
|
|
_comments.Clear();
|
|
_committedComments = 0;
|
|
}
|
|
}
|
|
|
|
private void ClearPendingComments()
|
|
{
|
|
if (_comments != null)
|
|
{
|
|
int startingComment = _committedComments + _preprocessorPendingCommentsCount;
|
|
_comments.RemoveRange(startingComment, _comments.Count - startingComment);
|
|
}
|
|
}
|
|
|
|
private static bool IsFirstTokenInLine(ReadOnlySpan<char> span, int startPos)
|
|
{
|
|
for (int pos = startPos; --pos > 0;)
|
|
{
|
|
switch (span[pos])
|
|
{
|
|
case '\r':
|
|
case '\n':
|
|
return true;
|
|
case ' ':
|
|
case '\t':
|
|
break;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get the next token
|
|
/// </summary>
|
|
/// <returns>Return the next token from the stream. If the end of stream is reached, a token type of None will be returned.</returns>
|
|
private UhtToken GetTokenInternal(bool enablePreprocessor)
|
|
{
|
|
ReadOnlySpan<char> span = _data.Span;
|
|
bool gotInlineComment = false;
|
|
Restart:
|
|
_preCurrentTokenInputLine = _inputLine;
|
|
_preCurrentTokenInputPos = _inputPos;
|
|
SkipWhitespaceAndCommentsInternal(ref gotInlineComment, enablePreprocessor);
|
|
_prevPos = _inputPos;
|
|
_prevLine = _inputLine;
|
|
|
|
UhtTokenType tokenType = UhtTokenType.EndOfFile;
|
|
int startPos = _inputPos;
|
|
int startLine = _inputLine;
|
|
|
|
char c = InternalGetChar(span);
|
|
if (c == 0)
|
|
{
|
|
}
|
|
|
|
else if (UhtFCString.IsAlpha(c) || c == '_')
|
|
{
|
|
|
|
for (; _inputPos < span.Length; ++_inputPos)
|
|
{
|
|
c = span[_inputPos];
|
|
if (!(UhtFCString.IsAlpha(c) || UhtFCString.IsDigit(c) || c == '_'))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (_inputPos - startPos >= UhtToken.MaxNameLength)
|
|
{
|
|
throw new UhtException(this, $"Identifier length exceeds maximum of {UhtToken.MaxNameLength}");
|
|
}
|
|
|
|
tokenType = UhtTokenType.Identifier;
|
|
}
|
|
|
|
// Check for any numerics
|
|
else if (IsNumeric(span, c))
|
|
{
|
|
// Integer or floating point constant.
|
|
|
|
bool isFloat = c == '.';
|
|
bool isHex = false;
|
|
|
|
// Ignore the starting sign for this part of the parsing
|
|
if (UhtFCString.IsSign(c))
|
|
{
|
|
c = InternalGetChar(span); // We know this won't fail since the code above checked the Peek
|
|
}
|
|
|
|
// Check for a hex valid
|
|
if (c == '0')
|
|
{
|
|
isHex = UhtFCString.IsHexMarker(InternalPeekChar(span));
|
|
if (isHex)
|
|
{
|
|
InternalGetChar(span);
|
|
}
|
|
}
|
|
|
|
// If we have a hex constant
|
|
if (isHex)
|
|
{
|
|
for (; _inputPos < span.Length && UhtFCString.IsHexDigit(span[_inputPos]); ++_inputPos)
|
|
{
|
|
}
|
|
}
|
|
|
|
// We have decimal/octal value or possibly a floating point value
|
|
else
|
|
{
|
|
|
|
// Skip all digits
|
|
for (; _inputPos < span.Length && UhtFCString.IsDigit(span[_inputPos]); ++_inputPos)
|
|
{
|
|
}
|
|
|
|
// If we have a '.'
|
|
if (_inputPos < span.Length && span[_inputPos] == '.')
|
|
{
|
|
isFloat = true;
|
|
++_inputPos;
|
|
|
|
// Skip all digits
|
|
for (; _inputPos < span.Length && UhtFCString.IsDigit(span[_inputPos]); ++_inputPos)
|
|
{
|
|
}
|
|
}
|
|
|
|
// If we have a 'e'
|
|
if (_inputPos < span.Length && UhtFCString.IsExponentMarker(span[_inputPos]))
|
|
{
|
|
isFloat = true;
|
|
++_inputPos;
|
|
|
|
// Skip any signs
|
|
if (_inputPos < span.Length && UhtFCString.IsSign(span[_inputPos]))
|
|
{
|
|
++_inputPos;
|
|
}
|
|
|
|
// Skip all digits
|
|
for (; _inputPos < span.Length && UhtFCString.IsDigit(span[_inputPos]); ++_inputPos)
|
|
{
|
|
}
|
|
}
|
|
|
|
// If we have a 'f'
|
|
if (_inputPos < span.Length && UhtFCString.IsFloatMarker(span[_inputPos]))
|
|
{
|
|
isFloat = true;
|
|
++_inputPos;
|
|
}
|
|
|
|
// Check for u/l markers
|
|
while (_inputPos < span.Length &&
|
|
(UhtFCString.IsUnsignedMarker(span[_inputPos]) || UhtFCString.IsLongMarker(span[_inputPos])))
|
|
{
|
|
++_inputPos;
|
|
}
|
|
}
|
|
|
|
if (_inputPos - startPos >= UhtToken.MaxNameLength)
|
|
{
|
|
throw new UhtException(this, $"Number length exceeds maximum of {UhtToken.MaxNameLength}");
|
|
}
|
|
|
|
tokenType = isFloat ? UhtTokenType.FloatConst : (isHex ? UhtTokenType.HexConst : UhtTokenType.DecimalConst);
|
|
}
|
|
|
|
// Escaped character constant
|
|
else if (c == '\'')
|
|
{
|
|
|
|
// We try to skip the character constant value. But if it is backslash, we have to skip another character
|
|
if (InternalGetChar(span) == '\\')
|
|
{
|
|
InternalGetChar(span);
|
|
}
|
|
|
|
char nextChar = InternalGetChar(span);
|
|
if (nextChar != '\'')
|
|
{
|
|
throw new UhtException(this, "Unterminated character constant");
|
|
}
|
|
|
|
tokenType = UhtTokenType.CharConst;
|
|
}
|
|
|
|
// String constant
|
|
else if (c == '"')
|
|
{
|
|
for (; ; )
|
|
{
|
|
char nextChar = InternalGetChar(span);
|
|
if (nextChar == '\r' || nextChar == '\n' || nextChar == 0)
|
|
{
|
|
throw new UhtException(this, "Unterminated character constant");
|
|
}
|
|
|
|
else if (nextChar == '\\')
|
|
{
|
|
nextChar = InternalGetChar(span);
|
|
if (nextChar == '\r' || nextChar == '\n' || nextChar == 0)
|
|
{
|
|
throw new UhtException(this, "Unterminated character constant");
|
|
}
|
|
}
|
|
|
|
else if (nextChar == '"')
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (_inputPos - startPos >= UhtToken.MaxStringLength)
|
|
{
|
|
throw new UhtException(this, $"String constant exceeds maximum of {UhtToken.MaxStringLength} characters");
|
|
}
|
|
|
|
tokenType = UhtTokenType.StringConst;
|
|
}
|
|
|
|
// Assume everything else is a symbol.
|
|
// Don't handle >> or >>>
|
|
else
|
|
{
|
|
{
|
|
if (_inputPos < span.Length)
|
|
{
|
|
char d = span[_inputPos];
|
|
if ((c == '<' && d == '<')
|
|
|| (c == '!' && d == '=')
|
|
|| (c == '<' && d == '=')
|
|
|| (c == '>' && d == '=')
|
|
|| (c == '+' && d == '+')
|
|
|| (c == '-' && d == '-')
|
|
|| (c == '+' && d == '=')
|
|
|| (c == '-' && d == '=')
|
|
|| (c == '*' && d == '=')
|
|
|| (c == '/' && d == '=')
|
|
|| (c == '&' && d == '&')
|
|
|| (c == '|' && d == '|')
|
|
|| (c == '^' && d == '^')
|
|
|| (c == '=' && d == '=')
|
|
|| (c == '*' && d == '*')
|
|
|| (c == '~' && d == '=')
|
|
|| (c == ':' && d == ':')
|
|
|| (c == '[' && d == '['))
|
|
{
|
|
++_inputPos;
|
|
}
|
|
}
|
|
|
|
// Comment processing while processing the preprocessor statements is complicated. When a preprocessor statement
|
|
// parsers, the tokenizer is doing some form of a PeekToken. And comments read ahead of the '#' will still be
|
|
// pending.
|
|
//
|
|
// 1) If the block is being skipped, we want to preserve any comments ahead of the token and eliminate all other comments found in the #if block
|
|
// 2) If the block isn't being skipped or in the case of things such as #pragma or #include, they were considered statements in the UHT
|
|
// and would result in the comments being dropped. In the new UHT, we just eliminate all pending tokens.
|
|
if (c == '#' && enablePreprocessor && _tokenPreprocessor != null && IsFirstTokenInLine(span, startPos))
|
|
{
|
|
_preprocessorPendingCommentsCount = _comments != null ? _comments.Count - _committedComments : 0;
|
|
UhtToken temp = new(tokenType, startPos, startLine, startPos, _inputLine, new StringView(_data[startPos.._inputPos]));
|
|
bool include = _tokenPreprocessor.ParsePreprocessorDirective(ref temp, true, out bool clearComments, out bool illegalContentsCheck);
|
|
if (!include)
|
|
{
|
|
++_commentsDisableCount;
|
|
int checkStateMachine = 0;
|
|
while (true)
|
|
{
|
|
if (IsEOF)
|
|
{
|
|
break;
|
|
}
|
|
UhtToken localToken = GetTokenInternal(false);
|
|
ReadOnlySpan<char> localValueSpan = localToken.Value.Span;
|
|
if (localToken.IsSymbol('#') && localValueSpan.Length == 1 && localValueSpan[0] == '#' && IsFirstTokenInLine(span, localToken.InputStartPos))
|
|
{
|
|
if (_tokenPreprocessor.ParsePreprocessorDirective(ref temp, false, out bool _, out bool scratchIllegalContentsCheck))
|
|
{
|
|
break;
|
|
}
|
|
illegalContentsCheck = scratchIllegalContentsCheck;
|
|
}
|
|
else if (illegalContentsCheck)
|
|
{
|
|
switch (checkStateMachine)
|
|
{
|
|
case 0:
|
|
ResetStateMachineCheck:
|
|
if (localValueSpan.CompareTo("UPROPERTY", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("UCLASS", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("USTRUCT", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("UENUM", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("UINTERFACE", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("UDELEGATE", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("UFUNCTION", StringComparison.Ordinal) == 0)
|
|
{
|
|
this.LogError($"'{localValueSpan.ToString()}' must not be inside preprocessor blocks, except for WITH_EDITORONLY_DATA");
|
|
}
|
|
else if (localValueSpan.CompareTo("void", StringComparison.Ordinal) == 0)
|
|
{
|
|
checkStateMachine = 1;
|
|
}
|
|
break;
|
|
|
|
case 1:
|
|
if (localValueSpan.CompareTo("Serialize", StringComparison.Ordinal) == 0)
|
|
{
|
|
checkStateMachine = 2;
|
|
}
|
|
else
|
|
{
|
|
checkStateMachine = 0;
|
|
goto ResetStateMachineCheck;
|
|
}
|
|
break;
|
|
|
|
case 2:
|
|
if (localValueSpan.CompareTo("(", StringComparison.Ordinal) == 0)
|
|
{
|
|
checkStateMachine = 3;
|
|
}
|
|
else
|
|
{
|
|
checkStateMachine = 0;
|
|
goto ResetStateMachineCheck;
|
|
}
|
|
break;
|
|
|
|
case 3:
|
|
if (localValueSpan.CompareTo("FArchive", StringComparison.Ordinal) == 0 ||
|
|
localValueSpan.CompareTo("FStructuredArchiveRecord", StringComparison.Ordinal) == 0)
|
|
{
|
|
this.LogError($"Engine serialization functions must not be inside preprocessor blocks, except for WITH_EDITORONLY_DATA");
|
|
checkStateMachine = 0;
|
|
}
|
|
else if (localValueSpan.CompareTo("FStructuredArchive", StringComparison.Ordinal) == 0)
|
|
{
|
|
checkStateMachine = 4;
|
|
}
|
|
else
|
|
{
|
|
checkStateMachine = 0;
|
|
goto ResetStateMachineCheck;
|
|
}
|
|
break;
|
|
|
|
case 4:
|
|
if (localValueSpan.CompareTo("::", StringComparison.Ordinal) == 0)
|
|
{
|
|
checkStateMachine = 5;
|
|
}
|
|
else
|
|
{
|
|
checkStateMachine = 0;
|
|
goto ResetStateMachineCheck;
|
|
}
|
|
break;
|
|
|
|
case 5:
|
|
if (localValueSpan.CompareTo("FRecord", StringComparison.Ordinal) == 0)
|
|
{
|
|
this.LogError($"Engine serialization functions must not be inside preprocessor blocks, except for WITH_EDITORONLY_DATA");
|
|
checkStateMachine = 0;
|
|
}
|
|
else
|
|
{
|
|
checkStateMachine = 0;
|
|
goto ResetStateMachineCheck;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
--_commentsDisableCount;
|
|
|
|
// Clear any extra pending comments at this time
|
|
ClearPendingComments();
|
|
}
|
|
_preprocessorPendingCommentsCount = 0;
|
|
|
|
// Depending on the type of directive and/or the #if expression, we clear comments.
|
|
if (clearComments)
|
|
{
|
|
ClearPendingComments();
|
|
gotInlineComment = false;
|
|
}
|
|
goto Restart;
|
|
}
|
|
}
|
|
tokenType = UhtTokenType.Symbol;
|
|
}
|
|
|
|
return new UhtToken(tokenType, startPos, startLine, startPos, _inputLine, new StringView(_data[startPos.._inputPos]));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Skip all leading whitespace and collect any comments
|
|
/// </summary>
|
|
private void SkipWhitespaceAndCommentsInternal(ref bool persistGotInlineComment, bool enablePreprocessor)
|
|
{
|
|
ReadOnlySpan<char> span = _data.Span;
|
|
|
|
bool gotNewlineBetweenComments = false;
|
|
bool gotInlineComment = persistGotInlineComment;
|
|
for (; ; )
|
|
{
|
|
uint c = InternalGetChar(span);
|
|
if (c == 0)
|
|
{
|
|
break;
|
|
}
|
|
else if (c == '\n')
|
|
{
|
|
gotNewlineBetweenComments |= gotInlineComment;
|
|
++_inputLine;
|
|
}
|
|
else if (c == '\r' || c == '\t' || c == ' ')
|
|
{
|
|
}
|
|
else if (c == '/')
|
|
{
|
|
uint nextChar = InternalPeekChar(span);
|
|
if (nextChar == '*')
|
|
{
|
|
if (enablePreprocessor)
|
|
{
|
|
ClearAllComments();
|
|
}
|
|
int commentStart = _inputPos - 1;
|
|
++_inputPos;
|
|
for (; ; )
|
|
{
|
|
char commentChar = InternalGetChar(span);
|
|
if (commentChar == 0)
|
|
{
|
|
if (enablePreprocessor)
|
|
{
|
|
ClearAllComments();
|
|
}
|
|
throw new UhtException(this, "End of header encountered inside comment");
|
|
}
|
|
else if (commentChar == '\n')
|
|
{
|
|
++_inputLine;
|
|
}
|
|
else if (commentChar == '*' && InternalPeekChar(span) == '/')
|
|
{
|
|
++_inputPos;
|
|
break;
|
|
}
|
|
}
|
|
if (enablePreprocessor)
|
|
{
|
|
AddComment(new StringView(_data[commentStart.._inputPos]));
|
|
}
|
|
}
|
|
else if (nextChar == '/')
|
|
{
|
|
if (gotNewlineBetweenComments)
|
|
{
|
|
gotNewlineBetweenComments = false;
|
|
if (enablePreprocessor)
|
|
{
|
|
ClearAllComments();
|
|
}
|
|
}
|
|
gotInlineComment = true;
|
|
int commentStart = _inputPos - 1;
|
|
++_inputPos;
|
|
|
|
// Scan to the end of the line
|
|
for (; ; )
|
|
{
|
|
char commentChar = InternalGetChar(span);
|
|
if (commentChar == 0)
|
|
{
|
|
//--Pos;
|
|
break;
|
|
}
|
|
if (commentChar == '\r')
|
|
{
|
|
}
|
|
else if (commentChar == '\n')
|
|
{
|
|
++_inputLine;
|
|
break;
|
|
}
|
|
}
|
|
if (enablePreprocessor)
|
|
{
|
|
AddComment(new StringView(_data[commentStart.._inputPos]));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
--_inputPos;
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
--_inputPos;
|
|
break;
|
|
}
|
|
}
|
|
persistGotInlineComment = gotInlineComment;
|
|
return;
|
|
}
|
|
|
|
private bool IsNumeric(ReadOnlySpan<char> span, char c)
|
|
{
|
|
// Check for [0..9]
|
|
if (UhtFCString.IsDigit(c))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Check for [+-]...
|
|
if (UhtFCString.IsSign(c))
|
|
{
|
|
if (_inputPos == span.Length)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Check for [+-][0..9]...
|
|
if (UhtFCString.IsDigit(span[_inputPos]))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Check for [+-][.][0..9]...
|
|
if (span[_inputPos] != '.')
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (_inputPos + 1 == span.Length)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (UhtFCString.IsDigit(span[_inputPos + 1]))
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
if (c == '.')
|
|
{
|
|
if (_inputPos == span.Length)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Check for [.][0..9]...
|
|
if (UhtFCString.IsDigit(span[_inputPos]))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fetch the next character in the input stream or zero if we have reached the end.
|
|
/// The current offset in the buffer is not advanced. The method does not support UTF-8
|
|
/// </summary>
|
|
/// <param name="span">The span containing the data</param>
|
|
/// <returns>Next character in the stream or zero</returns>
|
|
private char InternalPeekChar(ReadOnlySpan<char> span)
|
|
{
|
|
return _inputPos < span.Length ? span[_inputPos] : '\0';
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fetch the next character in the input stream or zero if we have reached the end.
|
|
/// The current offset in the buffer is advanced. The method does not support UTF-8.
|
|
/// </summary>
|
|
/// <param name="span">The span containing the data</param>
|
|
/// <returns>Next character in the stream or zero</returns>
|
|
private char InternalGetChar(ReadOnlySpan<char> span)
|
|
{
|
|
return _inputPos < span.Length ? span[_inputPos++] : '\0';
|
|
}
|
|
|
|
/// <summary>
|
|
/// If we have a current token, then reset the pending comments and input position back to before the token.
|
|
/// </summary>
|
|
private void ClearToken()
|
|
{
|
|
if (_hasToken)
|
|
{
|
|
_hasToken = false;
|
|
if (_comments != null && _comments.Count > _committedComments)
|
|
{
|
|
_comments.RemoveRange(_committedComments, _comments.Count - _committedComments);
|
|
}
|
|
_inputPos = _currentToken.UngetPos;
|
|
_inputLine = _currentToken.UngetLine;
|
|
}
|
|
}
|
|
|
|
private void AddComment(StringView comment)
|
|
{
|
|
_comments ??= new List<StringView>(4);
|
|
_comments.Add(comment);
|
|
}
|
|
#endregion
|
|
}
|
|
}
|