static TokenData ScanSingleChar(TextInputRange input, Token token)
{
input.Extend();
return(new TokenData {
Range = input.Clone(), Type = token
});
}
static TokenData ScanNewLine(TextInputRange input)
{
input.NewLine();
input.ExtendWhitespaces();
return(new TokenData {
Range = input.Clone(), Type = Token.NewLineIndentation
});
}
static TokenData ScanWhitespaces(TextInputRange input)
{
input.Extend();
input.ExtendWhitespaces();
return(new TokenData {
Range = input.Clone(), Type = Token.WhiteSpaceSeperator
});
}
static TokenData ScanComment(TextInputRange input)
{
var comment = CommentScanner.Scan(input);
Debug.Assert(comment);
return(new TokenData {
Range = input.Clone(), Type = Token.Comment
});
}
// Scan basic double quoted strings
public static IStringLiteral Scan(TextInputRange input)
{
var chr = input.EndChar;
if (!IsDoubleQuote(chr))
{
return(null);
}
input.Extend();
var result = new StringBuilder();
while (true)
{
chr = input.EndChar;
if (chr == '\0')
{
return(null); // file end or invalid input
}
if (IsDoubleQuote(chr))
{
break;
}
if (input.IsEndNewline)
{
input.NewLine();
}
else
{
if (!IsTab(chr) && char.IsControl(chr))
{
HandleControl(input);
}
else
{
if (IsBackslash(chr))
{
if (!HandleEscape(input, result))
{
return(null);
}
}
else
{
result.Append(chr);
input.Extend();
}
}
}
}
input.Extend();
return(new StringLiteral {
Content = result.ToString(), Range = input.Clone()
});
}
// this will scan the longest valid regular entry
public static IIdentifierLiteral Scan(TextInputRange input)
{
if (!IsStart(input))
{
input.Backtrack(); // dot might have been skipped
return(null);
}
do
{
input.Extend();
} while (IsContinue(input));
return(new IdentifierLiteral {
Content = input.Text, Range = input.Clone()
});
}
public static IEnumerable <TokenData> ScanFile(TextFile file)
{
var input = new TextInputRange {
File = file
};
while (true)
{
input.Collapse();
var chr = input.EndChar;
// ReSharper disable once SwitchStatementMissingSomeCases
switch (chr)
{
case '\0':
yield break;
case ' ':
case '\t':
yield return(ScanWhitespaces(input));
continue;
case '\n':
case '\r':
yield return(ScanNewLine(input));
continue;
case '#':
yield return(ScanComment(input));
continue;
case ',':
yield return(ScanSingleChar(input, Token.CommaSeparator));
continue;
case ';':
yield return(ScanSingleChar(input, Token.SemicolonSeparator));
continue;
case '[':
yield return(ScanSingleChar(input, Token.SquareBracketOpen));
continue;
case ']':
yield return(ScanSingleChar(input, Token.SquareBracketClose));
continue;
case '(':
yield return(ScanSingleChar(input, Token.BracketOpen));
continue;
case ')':
yield return(ScanSingleChar(input, Token.BracketClose));
continue;
case '"':
yield return(ScanStringLiteral(input));
continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
yield return(ScanNumberLiteral(input));
continue;
}
var identifierLiteral = IdentifierScanner.Scan(input);
if (identifierLiteral != null)
{
yield return(new TokenData {
Range = identifierLiteral.Range, Type = Token.IdentifierLiteral, Data = identifierLiteral
});
continue;
}
var operatorLiteral = OperatorScanner.Scan(input);
if (operatorLiteral != null)
{
yield return(new TokenData {
Range = operatorLiteral.Range, Type = Token.OperatorLiteral, Data = operatorLiteral
});
continue;
}
input.Extend();
yield return(new TokenData {
Range = input.Clone(), Type = Token.InvalidCharacter
});
}
}