public async Task Tokenize(IStream <IToken> tokenStream, ICharacterStream characterStream, string filename, int startLine, int startPosition)
{
this.Filename = filename;
this.Line = startLine;
this.Position = startPosition;
Queue <char> q = new Queue <char>();
ITokenizerState current = this.StartState;
ITokenizerState next = this.StartState;
bool handled = true;
ICharacter c = null;
while ((current != this.EndState) && (current != this.ErrorState))
{
current = next;
if (handled)
{
c = await characterStream.Get();
handled = false;
}
handled = current.Handle(c, tokenStream, q);
next = current.NextState(c.Kind);
}
}
private ReadOnlyCollection <Token> TryFindMarker(ICharacterStream stream)
{
var result = new List <Token>();
string?foundString = null;
var matches = MarkerCollection.LanguageMarkers
.Where(marker => stream.Remaining.StartsWith(marker.Start)).ToList();
if (matches.Any())
{
foundString = matches.Single().Start;
}
else
{
matches = MarkerCollection.LanguageMarkers
.Where(marker => stream.Remaining.StartsWith(marker.End)).ToList();
if (matches.Any())
{
foundString = matches.Single().End;
}
}
if (foundString != null)
{
result.AddRange(PopBuffer());
result.Add(new Token(TokenType.Marker, foundString));
stream.Skip(foundString.Length - 1);
}
return(result.AsReadOnly());
}
private ReadOnlyCollection <Token> TryFindSpecial(ICharacterStream stream)
{
var result = new List <Token>();
// Find start-of-line marker.
string[] patterns = { @">+", @"==+", @"[^\*](\*)[^\*]", @"[^-](-)[^->]", "----+" };
var remaining = stream.Remaining.ToString();
var index = remaining.IndexOf(ch => !char.IsWhiteSpace(ch));
if (index != -1)
{
var bestMatch = patterns
.Select(pattern => Regex.Match(remaining, pattern))
.Where(match => match.Success)
.Select(match => match.Groups[^ 1])
