public void Test_Common() { string data1 = null; Assert.Throws(typeof(ArgumentNullException), () => { new StringTokenizer(data1); }); char[] data2 = null; Assert.Throws(typeof(ArgumentNullException), () => { new StringTokenizer(data2); }); StringTokenizer strTok = new StringTokenizer("alpha beta 123 456.57, x 0x123F"); Assert.IsNotNull(strTok); strTok.RecognizeHex = true; strTok.IgnoreWhiteSpace = false; Assert.IsFalse(strTok.IgnoreWhiteSpace); char[] symChars = strTok.SymbolChars; strTok.SymbolChars = symChars; strTok.RecognizeDecimals = false; Assert.IsFalse(strTok.RecognizeDecimals); Token tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("alpha", tok.Value); Assert.AreEqual(5, strTok.Position); Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind); tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("beta", tok.Value); Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind); tok = strTok.Next(); Assert.AreEqual(TokenKind.Number, tok.Kind); Assert.AreEqual("123", tok.Value); Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind); tok = strTok.Next(); Assert.AreEqual(TokenKind.Number, tok.Kind); Assert.AreEqual("456", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Symbol, tok.Kind); Assert.AreEqual(".", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Number, tok.Kind); Assert.AreEqual("57", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Symbol, tok.Kind); Assert.AreEqual(",", tok.Value); Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind); tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("x", tok.Value); Assert.AreEqual(TokenKind.WhiteSpace, strTok.Next().Kind); tok = strTok.Next(); Assert.AreEqual(TokenKind.HexNumber, tok.Kind); Assert.AreEqual(true, strTok.RequireToken(TokenKind.HexNumber)); Assert.AreEqual("0x123F", tok.Value); // strTok = new StringTokenizer("alpha beta 123 456.57, x; \r\n \r \n \"test quote\" \"test \r \n quote2\""); Assert.IsNotNull(strTok); strTok.IgnoreWhiteSpace = true; Assert.IsTrue(strTok.IgnoreWhiteSpace); strTok.RecognizeDecimals = true; Assert.IsTrue(strTok.RecognizeDecimals); tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("alpha", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("beta", tok.Value); tok = strTok.Next(); Assert.IsTrue(strTok.RequireToken(TokenKind.Number)); Assert.AreEqual(TokenKind.Number, tok.Kind); Assert.AreEqual("123", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Number, tok.Kind); Assert.AreEqual("456.57", tok.Value); tok = strTok.Next(); strTok.RequestSymbol(','); Assert.AreEqual(TokenKind.Symbol, tok.Kind); Assert.AreEqual(",", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("x", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.Symbol, tok.Kind); Assert.AreEqual(";", tok.Value); Assert.AreEqual(1, tok.Line); Assert.Throws(typeof(Exception), () => { strTok.RequestSymbol('x'); }); tok = strTok.Next(); Assert.AreEqual(TokenKind.EOL, tok.Kind); Assert.AreEqual(1, tok.Line); tok = strTok.Next(); Assert.AreEqual(TokenKind.EOL, tok.Kind); Assert.AreEqual(2, tok.Line); tok = strTok.Next(); Assert.AreEqual(TokenKind.EOL, tok.Kind); Assert.AreEqual(3, tok.Line); tok = strTok.Next(); Assert.AreEqual(TokenKind.QuotedString, tok.Kind); Assert.AreEqual("\"test quote\"", tok.Value); Assert.AreEqual(4, tok.Line); tok = strTok.Next(); Assert.AreEqual(TokenKind.QuotedString, tok.Kind); Assert.AreEqual("\"test \r \n quote2\"", tok.Value); // strTok = new StringTokenizer("\t alpha 0x601 0b11000000001 alg_123 12345"); Assert.IsNotNull(strTok); strTok.IgnoreWhiteSpace = true; strTok.RecognizeHex = true; Assert.IsTrue(strTok.RecognizeHex); strTok.RecognizeBin = true; Assert.IsTrue(strTok.RecognizeBin); strTok.RecognizeIdents = true; Assert.IsTrue(strTok.RecognizeIdents); tok = strTok.Next(); Assert.IsNotNull(strTok.CurrentToken); Assert.AreEqual(TokenKind.Word, tok.Kind); Assert.AreEqual("alpha", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.HexNumber, tok.Kind); Assert.AreEqual("0x601", tok.Value); tok = strTok.Next(); Assert.AreEqual(TokenKind.BinNumber, tok.Kind); Assert.AreEqual("0b11000000001", tok.Value); Assert.AreEqual(" alg_123 12345", strTok.GetRest()); tok = strTok.Next(); Assert.AreEqual(TokenKind.Ident, tok.Kind); Assert.AreEqual("alg_123", tok.Value); Assert.Throws(typeof(Exception), () => { strTok.RequestInt(); }); tok = strTok.Next(); Assert.AreEqual(12345, strTok.RequestInt()); Assert.AreEqual("", strTok.GetRest()); }
private void lex() { Token tok = fTokenizer.CurrentToken; switch (tok.Kind) { case TokenKind.Number: fToken = ExpToken.tkNUMBER; fValue = ConvertHelper.ParseFloat(tok.Value, double.NaN); break; case TokenKind.Symbol: { switch (tok.Value[0]) { case '!': fToken = ExpToken.tkNOT; tok = fTokenizer.Next(); if (tok.Value[0] == '=') { fToken = ExpToken.tkNE; } else { return; } break; case '%': fToken = ExpToken.tkMOD; tok = fTokenizer.Next(); if (tok.Value[0] == '%') { fToken = ExpToken.tkPER; } else { return; } break; case '&': fToken = ExpToken.tkAND; break; case '(': fToken = ExpToken.tkLBRACE; break; case ')': fToken = ExpToken.tkRBRACE; break; case '*': fToken = ExpToken.tkMUL; tok = fTokenizer.Next(); if (tok.Value[0] == '*') { fToken = ExpToken.tkPOW; } else { return; } break; case '+': fToken = ExpToken.tkADD; break; case '-': fToken = ExpToken.tkSUB; break; case '/': fToken = ExpToken.tkDIV; break; case ';': fToken = ExpToken.tkSEMICOLON; break; case '<': fToken = ExpToken.tkLT; tok = fTokenizer.Next(); if (tok.Value[0] == '=') { fToken = ExpToken.tkLE; } else { return; } break; case '=': fToken = ExpToken.tkASSIGN; tok = fTokenizer.Next(); if (tok.Value[0] == '=') { fToken = ExpToken.tkEQ; } else { return; } break; case '>': fToken = ExpToken.tkGT; tok = fTokenizer.Next(); if (tok.Value[0] == '=') { fToken = ExpToken.tkGE; } else { return; } break; case '^': fToken = ExpToken.tkXOR; break; case '|': fToken = ExpToken.tkOR; break; case '~': fToken = ExpToken.tkINV; break; default: fToken = ExpToken.tkERROR; break; } } break; case TokenKind.Word: case TokenKind.Ident: fIdent = tok.Value; fToken = ExpToken.tkIDENT; break; case TokenKind.HexNumber: case TokenKind.BinNumber: try { fToken = ExpToken.tkNUMBER; fValue = (int)tok.ValObj; } catch { fToken = ExpToken.tkERROR; } break; case TokenKind.EOF: fToken = ExpToken.tkEOF; break; default: fToken = ExpToken.tkERROR; break; } fTokenizer.Next(); }