override public Token getTok()
{
SbToken test = new SbToken();
test.Append((char)context_.src.peek());
test.Append((char)context_.src.peek(1));
if(twoCharTokens_.Contains(test.ToString()))
{
context_.src.next();
context_.src.next();
return test.ToString();
}
SbToken tok = new SbToken();
tok.Append((char)context_.src.next());
if (oneCharTokens_.Contains(tok.ToString()))
return tok.ToString();
while(context_.currentState_.isPunctuation())
{
if (
isMultiLineComment() || isSingleLineComment() ||
isDoubleQuote() || isSingleQuote()
)
break;
tok.Append((char)context_.src.next());
}
return tok.ToString();
}
//----< keep extracting until get non-punctuator >---------------
/*
* Here is where we handle single char and two char special tokens
* as well as other punctuators.
*/
override public Token getTok()
{
// is this a two char special token?
SbToken test = new SbToken();
test.Append((char)context_.src.peek());
test.Append((char)context_.src.peek(1));
if(twoCharTokens_.Contains(test.ToString()))
{
context_.src.next(); // pop peeked char
context_.src.next(); // pop peeked char
return test.ToString();
}
// is this a single char special token?
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // pop first punctuator
if (oneCharTokens_.Contains(tok.ToString()))
return tok.ToString();
// not special token, so continue collecting punctuation chars
while(context_.currentState_.isPunctuation())
{
// check for other special cases starting in middle of punctuator
if (
isMultiLineComment() || isSingleLineComment() ||
isDoubleQuote() || isSingleQuote()
)
break;
tok.Append((char)context_.src.next());
}
return tok.ToString();
}
//----< keep extracting until get non-alphanum >-----------------
public override string getTok()
{
var tok = new SbToken();
tok.Append((char) context_.src.next()); // first is alphanum
while (isAlphaNum()) // stop when non-alphanum
tok.Append((char) context_.src.next());
return tok.ToString();
}
//----< keep extracting until get non-whitespace >---------------
public override string getTok()
{
var tok = new SbToken();
tok.Append((char) context_.src.next()); // first is WhiteSpace
while (context_.currentState_.isWhiteSpace()) // stop when non-WhiteSpace
tok.Append((char) context_.src.next());
return tok.ToString();
}
//----< keep extracting until get non-alphanum >-----------------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next());
while (isAlphaNum())
{
tok.Append((char)context_.src.next());
}
return tok.ToString();
}
//----< keep extracting until get non-whitespace >---------------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next());
while (context_.currentState_.isWhiteSpace())
{
tok.Append((char)context_.src.next());
}
return tok.ToString();
}
//----< keep extracting until get end quote >--------------------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // char is '\''
while (true)
{
char ch = (char)context_.src.next();
tok.Append(ch);
if (ch == '\'' && !isEscaped(tok.ToString()))
break;
}
return tok.ToString();
}
//----< keep extracting until get comment termintor >-----------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // char is /
tok.Append((char)context_.src.next()); // char is *
char ch = ' ', prevCh = ' ';
while (true) // stop when newline
{
prevCh = ch;
ch = (char)context_.src.next();
tok.Append(ch);
if (prevCh == '*' && ch == '/')
break;
}
return tok.ToString();
}
//----< keep extracting until get newline >--------------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // char is /
tok.Append((char)context_.src.next()); // char is /
char ch;
while (true) // stop when newline
{
ch = (char)context_.src.peek();
if (ch == '\n')
break;
tok.Append((char)context_.src.next());
}
return tok.ToString();
}
//----< return first char in src, as it must be a newline >------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // first is newline
return tok.ToString();
}
//----< keep extracting until get end quote >--------------------
override public Token getTok()
{
SbToken tok = new SbToken();
tok.Append((char)context_.src.next()); // char is "\"" or "@"
char nxt = (char)context_.src.peek();
if (nxt == '\"' && tok[0] == '@')
tok.Append((char)context_.src.next());
while (true)
{
char ch = (char)context_.src.next();
tok.Append(ch);
if (ch == '\"' && (!isEscaped(tok.ToString()) || tok[0] == '@'))
break;
}
return tok.ToString();
}