public void Test_Common()
{
string data1 = null;
Assert.Throws(typeof(ArgumentNullException), () => { new StringTokenizer(data1); });
char[] data2 = null;
Assert.Throws(typeof(ArgumentNullException), () => { new StringTokenizer(data2); });
StringTokenizer strTok = new StringTokenizer("alpha beta 123 456.57, x 0x123F");
Assert.IsNotNull(strTok);
strTok.RecognizeHex = true;
strTok.IgnoreWhiteSpace = false;
Assert.IsFalse(strTok.IgnoreWhiteSpace);
char[] symChars = strTok.SymbolChars;
strTok.SymbolChars = symChars;
strTok.RecognizeDecimals = false;
Assert.IsFalse(strTok.RecognizeDecimals);
Token tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("alpha", tok.Value);
Assert.AreEqual(5, strTok.Position);
Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("beta", tok.Value);
Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Number, tok.Kind);
Assert.AreEqual("123", tok.Value);
Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Number, tok.Kind);
Assert.AreEqual("456", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Symbol, tok.Kind);
Assert.AreEqual(".", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Number, tok.Kind);
Assert.AreEqual("57", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Symbol, tok.Kind);
Assert.AreEqual(",", tok.Value);
Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("x", tok.Value);
Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind);
tok = strTok.Next();
Assert.AreEqual(TokenKind.HexNumber, tok.Kind);
Assert.AreEqual(true, strTok.RequireToken(TokenKind.HexNumber));
Assert.AreEqual("0x123F", tok.Value);
//
strTok = new StringTokenizer("alpha beta 123 456.57, x; \r\n \r \n \"test quote\" \"test \r \n quote2\"");
Assert.IsNotNull(strTok);
strTok.IgnoreWhiteSpace = true;
Assert.IsTrue(strTok.IgnoreWhiteSpace);
strTok.RecognizeDecimals = true;
Assert.IsTrue(strTok.RecognizeDecimals);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("alpha", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("beta", tok.Value);
tok = strTok.Next();
Assert.IsTrue(strTok.RequireToken(TokenKind.Number));
Assert.AreEqual(TokenKind.Number, tok.Kind);
Assert.AreEqual("123", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Number, tok.Kind);
Assert.AreEqual("456.57", tok.Value);
tok = strTok.Next();
strTok.RequestSymbol(',');
Assert.AreEqual(TokenKind.Symbol, tok.Kind);
Assert.AreEqual(",", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("x", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.Symbol, tok.Kind);
Assert.AreEqual(";", tok.Value);
Assert.AreEqual(1, tok.Line);
Assert.Throws(typeof(Exception), () => { strTok.RequestSymbol('x'); });
tok = strTok.Next();
Assert.AreEqual(TokenKind.EOL, tok.Kind);
Assert.AreEqual(1, tok.Line);
tok = strTok.Next();
Assert.AreEqual(TokenKind.EOL, tok.Kind);
Assert.AreEqual(2, tok.Line);
tok = strTok.Next();
Assert.AreEqual(TokenKind.EOL, tok.Kind);
Assert.AreEqual(3, tok.Line);
tok = strTok.Next();
Assert.AreEqual(TokenKind.QuotedString, tok.Kind);
Assert.AreEqual("\"test quote\"", tok.Value);
Assert.AreEqual(4, tok.Line);
tok = strTok.Next();
Assert.AreEqual(TokenKind.QuotedString, tok.Kind);
Assert.AreEqual("\"test \r \n quote2\"", tok.Value);
//
strTok = new StringTokenizer("\t alpha 0x601 0b11000000001 alg_123 12345");
Assert.IsNotNull(strTok);
strTok.IgnoreWhiteSpace = true;
strTok.RecognizeHex = true;
Assert.IsTrue(strTok.RecognizeHex);
strTok.RecognizeBin = true;
Assert.IsTrue(strTok.RecognizeBin);
strTok.RecognizeIdents = true;
Assert.IsTrue(strTok.RecognizeIdents);
tok = strTok.Next();
Assert.IsNotNull(strTok.CurrentToken);
Assert.AreEqual(TokenKind.Word, tok.Kind);
Assert.AreEqual("alpha", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.HexNumber, tok.Kind);
Assert.AreEqual("0x601", tok.Value);
tok = strTok.Next();
Assert.AreEqual(TokenKind.BinNumber, tok.Kind);
Assert.AreEqual("0b11000000001", tok.Value);
Assert.AreEqual(" alg_123 12345", strTok.GetRest());
tok = strTok.Next();
Assert.AreEqual(TokenKind.Ident, tok.Kind);
Assert.AreEqual("alg_123", tok.Value);
Assert.Throws(typeof(Exception), () => { strTok.RequestInt(); });
tok = strTok.Next();
Assert.AreEqual(12345, strTok.RequestInt());
Assert.AreEqual("", strTok.GetRest());
}
private void lex()
{
Token tok = fTokenizer.CurrentToken;
switch (tok.Kind)
{
case TokenKind.Number:
fToken = ExpToken.tkNUMBER;
fValue = ConvertHelper.ParseFloat(tok.Value, double.NaN);
break;
case TokenKind.Symbol:
{
switch (tok.Value[0])
{
case '!':
fToken = ExpToken.tkNOT;
tok = fTokenizer.Next();
if (tok.Value[0] == '=')
{
fToken = ExpToken.tkNE;
}
else
{
return;
}
break;
case '%':
fToken = ExpToken.tkMOD;
tok = fTokenizer.Next();
if (tok.Value[0] == '%')
{
fToken = ExpToken.tkPER;
}
else
{
return;
}
break;
case '&':
fToken = ExpToken.tkAND;
break;
case '(':
fToken = ExpToken.tkLBRACE;
break;
case ')':
fToken = ExpToken.tkRBRACE;
break;
case '*':
fToken = ExpToken.tkMUL;
tok = fTokenizer.Next();
if (tok.Value[0] == '*')
{
fToken = ExpToken.tkPOW;
}
else
{
return;
}
break;
case '+':
fToken = ExpToken.tkADD;
break;
case '-':
fToken = ExpToken.tkSUB;
break;
case '/':
fToken = ExpToken.tkDIV;
break;
case ';':
fToken = ExpToken.tkSEMICOLON;
break;
case '<':
fToken = ExpToken.tkLT;
tok = fTokenizer.Next();
if (tok.Value[0] == '=')
{
fToken = ExpToken.tkLE;
}
else
{
return;
}
break;
case '=':
fToken = ExpToken.tkASSIGN;
tok = fTokenizer.Next();
if (tok.Value[0] == '=')
{
fToken = ExpToken.tkEQ;
}
else
{
return;
}
break;
case '>':
fToken = ExpToken.tkGT;
tok = fTokenizer.Next();
if (tok.Value[0] == '=')
{
fToken = ExpToken.tkGE;
}
else
{
return;
}
break;
case '^':
fToken = ExpToken.tkXOR;
break;
case '|':
fToken = ExpToken.tkOR;
break;
case '~':
fToken = ExpToken.tkINV;
break;
default:
fToken = ExpToken.tkERROR;
break;
}
}
break;
case TokenKind.Word:
case TokenKind.Ident:
fIdent = tok.Value;
fToken = ExpToken.tkIDENT;
break;
case TokenKind.HexNumber:
case TokenKind.BinNumber:
try {
fToken = ExpToken.tkNUMBER;
fValue = (int)tok.ValObj;
} catch {
fToken = ExpToken.tkERROR;
}
break;
case TokenKind.EOF:
fToken = ExpToken.tkEOF;
break;
default:
fToken = ExpToken.tkERROR;
break;
}
fTokenizer.Next();
}
