public bool isOperator(Tokenizer t)
{
string isThereMore = "";
string collector = "";
int i = 0;
while (i < maxLen)
{
collector += t.peek(i + 1);
foreach (var op in operators)
{
if (collector == op)
{
isThereMore = collector;
}
}
i++;
}
foreach (var op in operators)
{
OpLen = op.Length;
if (isThereMore == op)
{
return(true);
}
}
return(false);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "id";
token.position = t.currentPos;
token.lineNumber = t.lineNumber;
while (t.hasNext() && (Char.IsLetterOrDigit(t.peek()) || t.peek() == '_'))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "Multi Line Comments";
token.position = t.currentPos;
token.lineNumber = t.lineNumber;
while (t.hasNext() && (t.peek() != '*' || t.peek(2) != '/'))
{
token.value += t.next();
}
token.value += "*/";
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "One Line Comments";
token.position = t.currentPos;
token.lineNumber = t.lineNumber;
while (t.hasNext() && t.peek() != '\n')
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer tokenizer)
{
var token = new Token();
token.value = "";
token.type = "space";
token.position = tokenizer.currentPos + 1;
token.lineNumber = tokenizer.lineNumber;
while (tokenizer.hasNext() && Char.IsWhiteSpace(tokenizer.peek()))
{
token.value += tokenizer.next();
}
return(token);
}
public bool isOperator(Tokenizer t)
{
string isThereMore = "";
foreach (var op in operators)
{
isThereMore = "";
int i = 1;
while (i <= op.Length)
{
isThereMore += t.peek(i++);
}
OpLen = op.Length;
if (isThereMore == op)
{
return(true);
}
}
return(false);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
int count = 0;
token.value = "";
token.type = "Integer";
token.position = t.currentPos;
token.lineNumber = t.lineNumber;
while (t.hasNext() && (char.IsDigit(t.peek()) || t.peek() == '.' || t.peek() == 'f'))
{
if (t.peek() == '.' || t.peek() == 'f')
{
token.type = "float";
count++;
if (count == 3)
{
return(null);
}
}
if (t.peek() == 'f')
{
if (t.peek(2) != ' ')
{
return(null);
}
}
token.value += t.next();
}
Char lastchar = token.value[token.value.Length - 1];
if (lastchar != 'f' && token.type == "float")
{
token.value += 'f';
}
return(token);
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasNext() && Char.IsDigit(t.peek()));
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasNext() && t.peek() == '#');
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasNext() && bracts.Contains(t.peek().ToString()));
}
public override bool tokenizable(Tokenizer tokenizer)
{
return(tokenizer.hasNext() && Char.IsWhiteSpace(tokenizer.peek()));
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasNext() && (Char.IsLetter(t.peek()) || t.peek() == '_'));
}