public static IEnumerable<Token<DicTokenType>> Tokenize(string source, string data)
{
var reader = new StringeReader(data);
Chare currentChar;
while (!reader.EndOfStringe)
{
if (char.IsWhiteSpace((currentChar = reader.ReadChare()).Character)) continue;
switch (currentChar.Character)
{
case '#':
yield return new Token<DicTokenType>(DicTokenType.Directive, ReadRestOfLine(reader).Trim());
break;
case '@':
reader.ReadUntilAny('\n', '\r');
break;
case '>':
{
bool diffmark = reader.Eat('>');
yield return new Token<DicTokenType>(diffmark ? DicTokenType.DiffEntry : DicTokenType.Entry,
ReadRestOfLine(reader, diffmark).Trim());
break;
}
case '|':
yield return new Token<DicTokenType>(DicTokenType.Property, ReadRestOfLine(reader).Trim());
break;
default:
throw new InvalidDataException($"{source}: (Line {currentChar.Line}, Col {currentChar.Column}) Unexpected token: '{currentChar}'.");
}
}
yield return new Token<DicTokenType>(DicTokenType.EOF, "");
}
public static IEnumerable<Token<TokenType>> GenerateTokens(string input)
{
var reader = new StringeReader(input);
while (!reader.EndOfStringe)
{
yield return reader.ReadToken(Rules);
}
}
public IEnumerator<Token<TokenType>> GetEnumerator()
{
StringeReader reader = new StringeReader(text);
Token<TokenType> token;
while ((token = reader.ReadToken(lexerRules)).Identifier != TokenType.EOF)
yield return token;
yield return token; // this makes sure the EOF token is returned
}
/// <summary>
/// Generates beautiful tokens.
/// </summary>
/// <param name="name">The source name of the input.</param>
/// <param name="input">The input string to tokenize.</param>
/// <returns></returns>
public static IEnumerable<Token<R>> GenerateTokens(string name, Stringe input)
{
var reader = new StringeReader(input)
{
Origin = name
};
while (!reader.EndOfStringe)
{
yield return reader.ReadToken(Rules);
}
}
private static Stringe ReadRestOfLine(StringeReader reader, bool isDiffmark = false) => isDiffmark
? reader.ReadUntilAny('>', '\n', '\r', '#', '@')
: reader.ReadUntilAny('>', '|', '\n', '\r', '#', '@');