protected char Read()
{
var ch = Peek();
sourcePosition++;
if (ch != NullChar)
{
CurrentTokenChars.Append(ch);
if (ch == '\r' && Peek() != '\n')
{
CurrentLine++;
PositionOnLine = 0;
}
else if (ch == '\n')
{
CurrentLine++;
PositionOnLine = 0;
}
PositionOnLine++;
position++;
}
return(ch);
}
/// <summary>
/// Tokenizes the input.
/// </summary>
public void Tokenize(IReader reader)
{
this.reader = reader;
try
{
CurrentLine = 1;
PositionOnLine = 0;
LastToken = null;
LastTokenPosition = 0;
Tokens.Clear();
CurrentTokenChars.Clear();
TokenizeCore();
}
finally
{
reader.Dispose();
}
}
protected bool TokenizeInternal(string sourceText, Func <bool> readFunction)
{
this.sourceText = sourceText;
try
{
position = 0;
CurrentLine = 1;
PositionOnLine = 0;
LastToken = null;
LastTokenPosition = 0;
Tokens.Clear();
CurrentTokenChars.Clear();
return(readFunction());
}
catch (Exception ex) when(ex.Message == "Assertion failed!")
{
return(false);
}
}
/// <summary>
/// Returns the current char and advances to the next one.
/// </summary>
protected char Read()
{
var ch = reader.Read();
if (ch != NullChar)
{
CurrentTokenChars.Append(ch);
if (ch == '\r' && Peek() != '\n')
{
CurrentLine++;
PositionOnLine = 0;
}
else if (ch == '\n')
{
CurrentLine++;
PositionOnLine = 0;
}
}
PositionOnLine++;
return(ch);
}
/// <summary>
/// Creates the token.
/// </summary>
protected TToken CreateToken(TTokenType type, int charsFromEndToSkip = 0, Func <TToken, TokenError> errorProvider = null)
{
var t = NewToken();
t.LineNumber = CurrentLine;
t.ColumnNumber = Math.Max(0, PositionOnLine - DistanceSinceLastToken - 1);
t.StartPosition = LastTokenPosition;
t.Length = DistanceSinceLastToken - charsFromEndToSkip;
t.Type = type;
t.Text = CurrentTokenChars.ToString().Substring(0, DistanceSinceLastToken - charsFromEndToSkip);
Tokens.Add(t);
if (errorProvider != null)
{
t.Error = errorProvider(t);
}
CurrentTokenChars.Remove(0, t.Length);
LastTokenPosition = position - charsFromEndToSkip;
OnTokenFound(t);
return(LastToken = t);
}
/// <summary>
/// Creates the token.
/// </summary>
protected TToken CreateToken(TTokenType type, int charsFromEndToSkip = 0, Func <TToken, TokenError> errorProvider = null)
{
LastToken = new TToken()
{
LineNumber = CurrentLine,
ColumnNumber = Math.Max(0, PositionOnLine - DistanceSinceLastToken - 1),
StartPosition = LastTokenPosition,
Length = DistanceSinceLastToken - charsFromEndToSkip,
Type = type,
Text = CurrentTokenChars.ToString().Substring(0, DistanceSinceLastToken - charsFromEndToSkip)
};
Tokens.Add(LastToken);
if (errorProvider != null)
{
LastToken.Error = errorProvider(LastToken);
}
CurrentTokenChars.Remove(0, LastToken.Length);
LastTokenPosition = reader.Position - charsFromEndToSkip;
OnTokenFound(LastToken);
return(LastToken);
}
protected TToken CreateToken(TTokenType type, int charsFromEndToSkip = 0, Func <TToken, TokenError>?errorProvider = null)
{
var text = CurrentTokenChars.ToString().Substring(0, DistanceSinceLastToken - charsFromEndToSkip);
var t = NewToken(text,
type,
lineNumber: CurrentLine,
columnNumber: Math.Max(0, PositionOnLine - DistanceSinceLastToken - 1),
length: DistanceSinceLastToken - charsFromEndToSkip,
startPosition: LastTokenPosition
);
Tokens.Add(t);
if (errorProvider != null)
{
t.Error = errorProvider(t);
}
CurrentTokenChars.Remove(0, t.Length);
LastTokenPosition = position - charsFromEndToSkip;
OnTokenFound(t);
return(LastToken = t);
}
