public Diagnostic(Location location, string message)
{
Location = location;
Message = message;
}
public static List<Token> Tokenize(string source, Compilation compilation)
{
int index = 0;
Location location = new Location()
{
Column = 1,
Line = 1
};
List<Token> tokenization = new List<Token>();
while(true)
{
if (index >= source.Length)
break;
int max = -1;
TokenPattern maxPattern = default(TokenPattern);
foreach(TokenPattern pattern in tokens)
{
int current = pattern.Matches(source, index);
if (current > max)
{
max = current;
maxPattern = pattern;
}
}
if (maxPattern.Matches == null)
{
compilation.ReportDiagnostic(location, "No matching token.");
if(source[index] == '\n')
{
location.Column = 1;
++location.Line;
}
else
{
++location.Column;
}
index++;
continue;
}
tokenization.Add(
new Token(
maxPattern.Id,
source.Substring(index, max),
location
)
);
for(int i = 0; i < max; ++i)
{
if (source[index + i] == '\n')
{
location.Column = 1;
++location.Line;
}
else
{
++location.Column;
}
}
index += max;
}
return tokenization;
}
public void ReportDiagnostic(Location location, string message)
{
ReportDiagnostic(new Diagnostic(location, message));
}
public Token(TokenId id, string text, Location location)
{
Id = id;
Text = text;
Location = location;
}