public void TestKeywordMatcher()
{
KeywordMatcher matcher = new KeywordMatcher("invalid", TokenType.INVALID, Delimiters, false);
StringTokenizer data = new StringTokenizer("+=+//= invalid=invalidinvalid$=");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a basic delimiter keyword
Assert.AreEqual(Delimiters.Find(m => m.Keyword == "+=").MatchNext(data, ref streamPos, Log).Type, TokenType.AddAssign);
Assert.AreEqual(data.CurrentItem, "+");
data.Advance();
// Assert that the matched token contains the value
Assert.AreEqual(Delimiters.Find(m => m.Keyword == "/").MatchNext(data, ref streamPos, Log).Value, "/");
Assert.AreEqual(data.CurrentItem, "/");
data.Advance(3);
// Match against a keyword rather than short operator as well as delimiter
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "=");
data.Advance();
// Assert that non-delimiters will prevent a match
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "i");
data.Advance(14);
// Ensure EOF with keywords
Assert.IsNotNull(Delimiters.Find(m => m.Keyword == "$=").MatchNext(data, ref streamPos, Log));
Assert.IsTrue(data.AtEnd());
}
public void TestWordMatcher()
{
WordMatcher matcher = new WordMatcher(Delimiters);
StringTokenizer data = new StringTokenizer("test \"'test test2+end");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a basic word
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Ensure that string/name tokens are not considered
// as they are exempt from the delimiters.
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
data.Advance();
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
data.Advance();
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "test");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match a word containing a number
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "test2");
Assert.AreEqual(data.CurrentItem, "+");
data.Advance();
// Ensure token type and EOF handling
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.Word);
Assert.IsTrue(data.AtEnd());
}
public void TestNumberMatcher()
{
NumberMatcher matcher = new NumberMatcher(Delimiters);
StringTokenizer data = new StringTokenizer("1 454 12h 0xa 0x 0xA12 0x1G 1.0 0.33 0x22.3 123.13a 1xA");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a single integer
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "1");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match several chars
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.IntegerNumber);
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Try to match a number that is actually a word
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(4);
// Match a hexadecimal number
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// fail to match a partial hex number
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(3);
// Match and assert value of hexadecimal number converted to decimal
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, (0xA12).ToString("D"));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// fail to match a malformed hex number
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(5);
// Match a floating point number
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.FloatingNumber);
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match a float and assert value
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "0.33");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Fail parsing a malformed hex-float
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(7);
// Fail parsing a malformed float-hex
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(8);
// Fail parsing a malformed hex prefix
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(3);
Assert.IsTrue(data.AtEnd());
}
public void TestWhiteSpaceMatcher()
{
WhiteSpaceMatcher matcher = new WhiteSpaceMatcher();
StringTokenizer data = new StringTokenizer(" , \n test ");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a single whitespace
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.WhiteSpace);
Assert.AreEqual(data.CurrentItem, ",");
data.Advance();
// Match a longer mixed whitespace and check that all of it is discarded
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log).Value);
Assert.AreEqual(data.CurrentItem, "t");
data.Advance(4);
// Match EOF whitespace
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.IsTrue(data.AtEnd());
}
public void TestKeywordMatcher()
{
KeywordMatcher matcher = new KeywordMatcher("invalid", TokenType.INVALID, Delimiters, false);
StringTokenizer data = new StringTokenizer("+=+//= invalid=invalidinvalid$=");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a basic delimiter keyword
Assert.AreEqual(Delimiters.Find(m => m.Keyword == "+=").MatchNext(data, ref streamPos, Log).Type, TokenType.AddAssign);
Assert.AreEqual(data.CurrentItem, "+");
data.Advance();
// Assert that the matched token contains the value
Assert.AreEqual(Delimiters.Find(m => m.Keyword == "/").MatchNext(data, ref streamPos, Log).Value, "/");
Assert.AreEqual(data.CurrentItem, "/");
data.Advance(3);
// Match against a keyword rather than short operator as well as delimiter
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "=");
data.Advance();
// Assert that non-delimiters will prevent a match
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "i");
data.Advance(14);
// Ensure EOF with keywords
Assert.IsNotNull(Delimiters.Find(m => m.Keyword == "$=").MatchNext(data, ref streamPos, Log));
Assert.IsTrue(data.AtEnd());
}
public void TestWordMatcher()
{
WordMatcher matcher = new WordMatcher(Delimiters);
StringTokenizer data = new StringTokenizer("test \"'test test2+end");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a basic word
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Ensure that string/name tokens are not considered
// as they are exempt from the delimiters.
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
data.Advance();
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
data.Advance();
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "test");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match a word containing a number
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "test2");
Assert.AreEqual(data.CurrentItem, "+");
data.Advance();
// Ensure token type and EOF handling
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.Word);
Assert.IsTrue(data.AtEnd());
}
public void TestWhiteSpaceMatcher()
{
WhiteSpaceMatcher matcher = new WhiteSpaceMatcher();
StringTokenizer data = new StringTokenizer(" , \n test ");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a single whitespace
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.WhiteSpace);
Assert.AreEqual(data.CurrentItem, ",");
data.Advance();
// Match a longer mixed whitespace and check that all of it is discarded
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log).Value);
Assert.AreEqual(data.CurrentItem, "t");
data.Advance(4);
// Match EOF whitespace
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.IsTrue(data.AtEnd());
}
public void TestNumberMatcher()
{
NumberMatcher matcher = new NumberMatcher(Delimiters);
StringTokenizer data = new StringTokenizer("1 454 12h 0xa 0x 0xA12 0x1G 1.0 0.33 0x22.3 123.13a 1xA");
SourcePosition streamPos = new SourcePosition(0, 0, 0);
// Match a single integer
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "1");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match several chars
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.IntegerNumber);
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Try to match a number that is actually a word
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(4);
// Match a hexadecimal number
Assert.IsNotNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// fail to match a partial hex number
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(3);
// Match and assert value of hexadecimal number converted to decimal
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, (0xA12).ToString("D"));
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// fail to match a malformed hex number
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(5);
// Match a floating point number
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Type, TokenType.FloatingNumber);
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Match a float and assert value
Assert.AreEqual(matcher.MatchNext(data, ref streamPos, Log).Value, "0.33");
Assert.AreEqual(data.CurrentItem, " ");
data.Advance();
// Fail parsing a malformed hex-float
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "0");
data.Advance(7);
// Fail parsing a malformed float-hex
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(8);
// Fail parsing a malformed hex prefix
Assert.IsNull(matcher.MatchNext(data, ref streamPos, Log));
Assert.AreEqual(data.CurrentItem, "1");
data.Advance(3);
Assert.IsTrue(data.AtEnd());
}