public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "CommentTokenizerMuitLines";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
token.value += t.next();
token.value += t.next();
//while (t.hasMore() && (t.peek() != '*')&& (t.peek(2) != '/'))
while (t.hasMore())
{
if ((t.peek() + "" + t.peek(2) == "*/"))
{
token.value += t.next();
token.value += t.next();
break;
}
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
int counter = 0;
Token token = new Token();
token.value += t.next(); // add the #
token.type = "hex-color";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
while (t.hasMore() &&
!char.IsWhiteSpace(t.peek()) &&
counter < 6)
{
if (!"0123456789abcdefABCDEF".Contains(t.peek()))
{
// has at least one hex value after #
if (counter > 0)
{
return(token);
}
else
{
t.currentPostion = token.position;
return(null);
}
}
counter++;
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "int";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
while (t.hasMore() &&
!char.IsWhiteSpace(t.peek()))
{
if (char.IsDigit(t.peek()))
{
token.value += t.next();
}
else if (t.peek() == '.')
{
if (token.type == "decimal")
{
return(token);
}
token.type = "decimal";
token.value += t.next();
}
}
// add 0 to the end if there is not zero !
token.value = token.value[token.value.Length - 1] == '.'
? (token.value += '0')
: token.value;
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.type = "single quotation";
token.value = "";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
int count = 0;
while (t.hasMore() && (Char.IsDigit(t.peek()) || t.peek() == '.'))
{
if (count <= 1)
{
if (t.peek() == '.')
{
++count;
token.value += t.next();
}
else
{
token.value += t.next();
}
}
else if (count >= 2)
{
token.value = "Error, there is more than one decimal.";
return(token);
}
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.type = "single quotation";
token.value = "";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
int count = 0;
while (t.hasMore() && (Char.IsLetterOrDigit(t.peek()) || t.peek() == '\''))
{
if (t.peek() == '\'')
{
token.value += t.next();
count++;
if (count == 2)
{
return(token);
}
}
else
{
token.value += t.next();
}
}
if (count % 2 != 0)
{
token.value = "Error, uneven count of quotations.";
}
return(token);
}
public override Token tokenize(Tokenizer tokenizer)
{
bool counter = false;
// "122.22 hi everyone 1.2 1."
Token token = new Token();
token.value = "";
token.type = "Decimal";
token.position = tokenizer.currentPosition;
token.lineNumber = tokenizer.lineNumber;
while (tokenizer.hasMore() && (char.IsDigit(tokenizer.peek()) || tokenizer.peek() == '.'))
{
if (char.IsDigit(tokenizer.peek()))
{
token.value += tokenizer.next();
}
else if (tokenizer.peek() == '.' && counter == false)
{
token.value += tokenizer.next();
counter = true;
}
else
{
//Console.WriteLine("error");
//throw new Exception("error");
tokenizer.next();
//return null;
}
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "Double Quot";
pos1 = t.currentPostion;
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
isClosed1 = false;
while (t.hasMore())
{
token.value += t.next();
if (t.peek() == ('\"'))
{
isClosed1 = true;
token.value += t.next();
break;
}
}
if (isClosed1)
{
return(token);
}
else
{
t.currentPostion = pos1;
token.value = "";
if (t.hasMore())
{
t.next();
return(token);
}
while (t.hasMore())
{
token.value += t.next();
if (t.peek() == '\"')
{
isClosed1 = true;
token.value += t.next();
break;
}
}
return(token);
}
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "Single Quot";
pos = t.currentPostion;
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
isClosed = false;
while (t.hasMore())
{
token.value += t.next();
if (t.peek() == char.Parse("'"))
{
isClosed = true;
token.value += t.next();
break;
}
}
if (isClosed)
{
return(token);
}
else
{
t.currentPostion = pos;
token.value = "";
if (t.hasMore())
{
t.next();
return(null);
}
while (t.hasMore())
{
token.value += t.next();
if (t.peek() == char.Parse("'"))
{
isClosed = true;
token.value += t.next();
break;
}
}
return(token);
}
}
public override bool tokenizable(Tokenizer t)
{
//Console.WriteLine("-------------hi-----------------");
//Console.WriteLine(t.peek());
//Console.WriteLine(t.peek(2));
//Console.WriteLine(t.peek() == '/');
//Console.WriteLine("------------------------------");
return(t.hasMore() && (t.peek() == '/') && (t.peek(2) == '/'));
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.type = "math Operator";
token.value = "";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
while (t.hasMore() && (t.peek() == '+' || t.peek() == '-' || t.peek() == '*' || t.peek() == '%' || t.peek() == '='))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.type = "whitespace";
token.value = "";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
while (t.hasMore() && Char.IsWhiteSpace(t.peek()))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "hashtag";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
while (t.hasMore() && (Char.IsLetterOrDigit(t.peek()) || t.peek() == '#'))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.type = "comment";
token.value = "";
token.position = t.currentPosition;
token.lineNumber = t.lineNumber;
while (t.hasMore() && t.peek() == '/')
{
token.value += t.next();
continue;
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "Comment";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
while (t.hasMore() && (t.peek() != '\n'))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value += t.next();
token.type = "color-hash";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
while (t.hasMore() &&
!char.IsWhiteSpace(t.peek()) &&
"0123456789abcdefABCDEF".Contains(t.peek()))
{
token.value += t.next();
}
return(token);
}
public override Token tokenize(Tokenizer tokenizer)
{
Token token = new Token();
token.value = "";
token.type = "HTMLTag";
token.position = tokenizer.currentPosition;
token.lineNumber = tokenizer.lineNumber;
while (tokenizer.hasMore() && (char.IsLetter(tokenizer.peek()) || "</>".Contains(tokenizer.peek())))
{
token.value += tokenizer.next();
if (tokenizer.peek() == '>')
{
token.value += tokenizer.next();
break;
}
}
// "<Header> Hi everyone <Div> inside a Div </Div> </Header>"
return(token);
}
public override Token tokenize(Tokenizer t)
{
Token token = new Token();
token.value = "";
token.type = "int";
token.position = t.currentPostion;
token.lineNumber = t.lineNumber;
while (t.hasMore() &&
!char.IsWhiteSpace(t.peek()))
{
if (char.IsDigit(t.peek())) // if token.type == "decimal" then throw error
{
token.value += t.next();
}
else if (t.peek() == '.')
{
token.type = "decimal";
token.value += t.next();
}
}
return(token);
}
public override bool tokenizable(Tokenizer tokenizer)
{
return(tokenizer.hasMore() && tokenizer.peek() == '<');
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore() && t.peek() == '#' && char.IsLetterOrDigit(t.peek(2)));
}
public bool isString(Tokenizer t)
{
return(t.hasMore());
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore() && t.peek() == '\"');
}
public bool isCommentStar(Tokenizer t)
{
return(t.hasMore());
}
public bool isComment(Tokenizer t)
{
return(t.hasMore() && (t.peek() != '\n') && (t.peek() == '/') || char.IsLetterOrDigit(t.peek()) || t.peek() == ' ');
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore() && char.IsWhiteSpace(t.peek()));
}
public bool isElse(Tokenizer t)
{
return(t.hasMore() && !Char.IsWhiteSpace(t.peek()) && (t.peek() != '/' && t.peek(2) != '/'));
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore() && char.IsDigit(t.peek()));
}
public bool isClass(Tokenizer t)
{
return(t.hasMore() && (char.IsLetterOrDigit(t.peek()) || t.peek() == '_'));
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore() && (t.peek() == '/' && t.peek(2) == '*'));
}
public bool isNumber(Tokenizer t)
{
return(t.hasMore() && (char.IsDigit(t.peek()) || t.peek() == '.'));
}
public override bool tokenizable(Tokenizer t)
{
return(t.hasMore());
}