/// <summary>
/// Gets tokens for grammar.
/// </summary>
/// <param name="text">A text for which tokens will be returned.</param>
/// <param name="state">A state for lexer.</param>
/// <returns>An enumerable of tokens.</returns>
public LexerResult GetTokens(string text, TLexerState state = null)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
var result = new LexerResult();
try
{
int currentTokenIndex = 0;
var lineProvider = new LexerLineNumberProvider(text);
if (state != null)
{
state.LineNumber = 0;
}
while (currentTokenIndex < text.Length)
{
if (state != null)
{
var currentLineIndex = lineProvider.GetLineForIndex(currentTokenIndex);
state.NewLine = state.LineNumber != currentLineIndex;
state.LineNumber = currentLineIndex;
state.StartColumnIndex = lineProvider.GetColumnForIndex(currentTokenIndex);
}
if (FindBestRule(text, currentTokenIndex, state, out var tokenType, out var returnToken, out var bestMatch, out var length))
{
if (returnToken)
{
if (state != null)
{
state.PreviousReturnedTokenType = tokenType;
}
result.Tokens.Add(new Token(
(int)tokenType,
bestMatch,
state?.LineNumber ?? 0,
state?.StartColumnIndex ?? 0,
null));
}
currentTokenIndex += length;
}
else
{
throw new LexerException(
"Can't get next token from text",
new SpiceLineInfo
{
LineNumber = state?.LineNumber ?? 0,
StartColumnIndex = state?.StartColumnIndex ?? 0,
});
}
}
/// <summary>
/// Gets tokens for grammar.
/// </summary>
/// <param name="text">A text for which tokens will be returned.</param>
/// <param name="state">A state for lexer.</param>
/// <returns>An enumerable of tokens.</returns>
public LexerResult GetTokens(string text, TLexerState state = null)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
var result = new LexerResult();
try
{
int currentTokenIndex = 0;
var lineProvider = new LexerLineInfoProvider(text);
if (state != null)
{
state.LineNumber = 0;
}
while (currentTokenIndex < text.Length)
{
if (state != null)
{
var currentLineIndex = lineProvider.GetLineForIndex(currentTokenIndex);
state.NewLine = state.LineNumber != currentLineIndex;
state.LineNumber = currentLineIndex;
state.StartColumnIndex = lineProvider.GetColumnForIndex(currentTokenIndex);
}
if (FindBestTokenRule(text, currentTokenIndex, state, out var bestTokenRule, out var bestMatch))
{
var tokenActionResult = bestTokenRule.ReturnDecisionProvider(state, bestMatch.Value);
if (tokenActionResult == LexerRuleReturnDecision.ReturnToken)
{
if (state != null)
{
state.PreviousReturnedTokenType = bestTokenRule.TokenType;
}
result.Tokens.Add(new Token(
bestTokenRule.TokenType,
bestMatch.Value,
state?.LineNumber ?? 0,
state?.StartColumnIndex ?? 0,
null));
}
currentTokenIndex += bestMatch.Length;
}
else
{
bool matched = false;
var textForDynamicRules = text.Substring(currentTokenIndex);
foreach (LexerDynamicRule dynamicRule in Grammar.DynamicRules)
{
bool ruleMatch = textForDynamicRules.StartsWith(dynamicRule.Prefix);
if (ruleMatch)
{
var dynamicResult = dynamicRule.Action(textForDynamicRules, state);
result.Tokens.Add(
new Token(
dynamicRule.TokenType,
dynamicResult.Item1,
state?.LineNumber ?? 0,
state?.StartColumnIndex ?? 0,
null));
currentTokenIndex += dynamicResult.Item2;
matched = true;
}
}
if (!matched)
{
throw new LexerException("Can't get next token from text",
new SpiceLineInfo
{
LineNumber = state?.LineNumber ?? 0,
StartColumnIndex = state?.StartColumnIndex ?? 0
});
}
}
}
