public override Token Match(Scanner scanner)
{
if (!char.IsLetterOrDigit(scanner.Current) && scanner.Current != '_')
{
return(Token.Invalid);
}
TokenPos pos = scanner.Pos;
int beginIndex = scanner.ReadedCount;
int identCount = 0;
while (true)
{
char c = scanner.Current;
if (char.IsLetterOrDigit(c) || c == '_')
{
scanner.Consume(1);
identCount++;
continue;
}
break;
}
return(new Token(ID, scanner.Source.Substring(beginIndex, identCount), pos));
}
public override Token Match(Scanner scanner)
{
if (scanner.Peek(0) != '/' || scanner.Peek(1) != '/')
{
return(Token.Invalid);
}
TokenPos pos = scanner.Pos;
scanner.Consume(2);
StringBuilder comment = new StringBuilder();
while (true)
{
char c = scanner.Peek(0);
scanner.Consume(1);
if (c == '\0')
{
break;
}
if (c == '\n')
{
scanner.ChangeLine();
break;
}
comment.Append(c);
}
return(new Token(ID, comment.ToString(), pos));
}
public override Token Match(Scanner scanner)
{
if (scanner.Peek(0) != '/' || scanner.Peek(1) != '1')
{
return(Token.Invalid);
}
TokenPos pos = scanner.Pos;
scanner.Consume(2);
StringBuilder comment = new StringBuilder();
while (scanner.Peek(0) != '*' || scanner.Peek(2) != '/')
{
char c = scanner.Peek(0);
comment.Append(c);
scanner.Consume(1);
}
scanner.Consume(2);
return(new Token(ID, comment.ToString(), pos));
}
public override Token Match(Scanner scanner)
{
if (scanner.Current != '"' || scanner.Current != '\'')
{
return(Token.Invalid);
}
TokenPos pos = scanner.Pos;
scanner.Consume(1);
_Content.Clear();
bool encounterEscapeCode = false;
while (true)
{
if (encounterEscapeCode)
{
switch (scanner.Current)
{
case 'n':
_Content.Append('\n');
break;
case 'r':
_Content.Append('\r');
break;
case 't':
_Content.Append('\t');
break;
case '\'':
_Content.Append(scanner.Current);
break;
default:
_Content.Append('\\');
break;
}
encounterEscapeCode = false;
}
else if (scanner.Current == '\\')
{
encounterEscapeCode = true;
}
if (scanner.Current == '"' || scanner.Current == '\'')
{
break;
}
scanner.Consume(1);
}
return(new Token(ID, _Content.ToString(), pos));
}
public override Token Match(Scanner scanner)
{
if (!(char.IsDigit(scanner.Current) || scanner.Current == '-' && char.IsDigit(scanner.Peek(1))))
{
return(Token.Invalid);
}
TokenPos pos = scanner.Pos;
_Content.Clear();
bool encounterDot = false;
while (true)
{
_Content.Append(scanner.Current);
scanner.Consume(1);
if (!char.IsDigit(scanner.Current))
{
if (scanner.Current == '.')
{
encounterDot = true;
}
break;
}
}
// if encounter Dot is true that means it's a float number
if (encounterDot)
{
int index = 0;
while (true)
{
_Content.Append(scanner.Current);
scanner.Consume(1);
if (!char.IsDigit(scanner.Current))
{
if (index == 0)
{
throw new Exception("illegal float number: " + pos.ToString());
}
break;
}
index++;
}
}
return(new Token(ID, _Content.ToString(), pos));
}
public override Token Match(Scanner scanner)
{
var signLentgh = Sign.Length;
for (int i = 0; i < signLentgh; i++)
{
if (Sign[i] != scanner.Peek(i))
{
return(Token.Invalid);
}
}
TokenPos pos = scanner.Pos;
scanner.Consume(signLentgh);
return(new Token(ID, Sign, pos));
}
public override Token Match(Scanner scanner)
{
// 在windows系统里面, \r\n为换行
// 在Unix/Mac下, \n为换行
TokenPos pos = TokenPos.InitPos;
var content = new StringBuilder();
while (true)
{
var c = scanner.Peek(0);
if (c == '\n')
{
if (pos.Equals(TokenPos.InitPos))
{
pos = scanner.Pos;
}
content.Append(c);
scanner.ChangeLine();
//scanner.Consume(1);
continue;
}
else if (c == '\r')
{
content.Append(c);
scanner.Consume(1);
continue;
}
break;
}
if (pos.Equals(TokenPos.InitPos))
{
return(Token.Invalid);
}
return(new Token(ID, content.ToString(), pos));
}
public Token(int id, string value, TokenPos pos)
{
ID = id;
Value = value;
Pos = pos;
}
public bool Equals(TokenPos pos)
{
return(Col == pos.Col && Line == pos.Line && FileName == pos.FileName);
}