public void WalksThroughTheInput()
{
using var t = new Tokeniser("static field let");
t.GetNextToken().Value.Should().Be("static");
t.GetNextToken().Value.Should().Be("field");
t.GetNextToken().Value.Should().Be("let");
}
public void DoesNotGetFooledByDoubleSlashesInStringLiterals()
{
using var t = new Tokeniser("return \"// this is not a comment\";\n");
t.GetNextToken().Value.Should().Be("return");
t.GetNextToken().Value.Should().Be("// this is not a comment");
t.GetNextToken().Value.Should().Be(";");
t.GetNextToken().Should().BeNull();
}
public void SkipsOverWhitespace()
{
using var t = new Tokeniser(" static field\nlet \r\n\tclass");
t.GetNextToken().Value.Should().Be("static");
t.GetNextToken().Value.Should().Be("field");
t.GetNextToken().Value.Should().Be("let");
t.GetNextToken().Value.Should().Be("class");
}
public void IgnoresLineCommentsAtTheEndOfLines()
{
using var t = new Tokeniser("return; // this is a comment\n}");
t.GetNextToken().Value.Should().Be("return");
t.GetNextToken().Value.Should().Be(";");
t.GetNextToken().Value.Should().Be("}");
t.GetNextToken().Should().BeNull();
}
public void WorksWhenPassedAStream()
{
byte[] bytes = Encoding.UTF8.GetBytes("static field\n");
using var stream = new MemoryStream(bytes);
using var t = new Tokeniser(stream);
t.GetNextToken().Value.Should().Be("static");
t.GetNextToken().Value.Should().Be("field");
t.GetNextToken().Should().BeNull();
}
public void RecognisesAllSymbols()
{
using var t = new Tokeniser("{ } ( ) [ ] . , ; + - * / & | < > = ~");
Token token = t.GetNextToken();
while (token != null)
{
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
}
}
public void RecognisesAllKeywords()
{
using var t = new Tokeniser("class constructor function method field static var int char boolean void true false null this let do if else while return");
Token token = t.GetNextToken();
while (token != null)
{
token.Type.Should().Be(NodeType.Keyword);
token = t.GetNextToken();
}
}
public void RejectsIntegersContainingStrings()
{
using var t = new Tokeniser("123four5");
Token token = t.GetNextToken();
token.Type.Should().NotBe(NodeType.IntegerConstant);
}
public DictionaryToken(Tokeniser tokeniser, ObjectId?objectId) : base(tokeniser, objectId)
{
// <<
// /Name1 123
// /Name2 [(string) (array) 123]
// /Name3 <</subDictionaryName1 123 /subDictionaryName2 true>>
// /Name4 (another string)
// /Name5 <112233EE>
// >>
this.tokeniser = tokeniser;
var b = tokeniser.SkipWhiteSpace();
if (b != '<' || tokeniser.GetNextByte() != '<')
{
throw tokeniser.Exception($"illegal dictionary format, leading characters '<<' expected, but was'{(char)b}{(char)tokeniser.LookaheadByte()}'.");
}
//parse key
tokens = new Dictionary <string, Token>();
tokeniser.GetNextByte();
b = tokeniser.SkipWhiteSpace();
while (b != '>' && tokeniser.LookaheadByte() != '>')
{
if (b != '/')
{
throw tokeniser.Exception($"Invalid dictionary format, '/' expected as leading character for dictionary key name, but was {(char)b}.");
}
var key = new NameToken(tokeniser, null);
var value = tokeniser.GetNextToken();
if (key.Value == "Type" && value is NameToken typeNameToken)
{
Type = typeNameToken.Value;
}
if (tokens.TryGetValue(key.Value, out var existingToken))
{
if (existingToken is ArrayToken existingArrayToken)
{
existingArrayToken.Add(value);
}
else
{
tokens[key.Value] = new ArrayToken(tokeniser, existingToken)
{
value
};
}
}
else
{
tokens.Add(key.Value, value);
}
b = tokeniser.SkipWhiteSpace();
}
tokeniser.GetNextByte();
tokeniser.GetNextByte();
StreamStartIndex = tokeniser.GetStreamStartIndex(this, out var length);
Length = length;
keys = tokens.Keys.ToArray();
}
public ArrayToken(Tokeniser tokeniser, ObjectId?objectId) : base(tokeniser, objectId)
{
//[/someName false -0 (string)]
this.tokeniser = tokeniser;
tokens = new List <Token>();
var b = tokeniser.SkipWhiteSpace();
if (b != '[')
{
throw tokeniser.Exception($"illegal array format, leading character '[' expected but was {(char)b}.");
}
b = tokeniser.GetNextByte();
while (b != ']')
{
var token = tokeniser.GetNextToken(isThrowExceptionWhenError: false);
if (token != null)
{
tokens.Add(token);
b = tokeniser.SkipWhiteSpace();
}
else
{
b = tokeniser.GetByte();
if (b != ']')
{
throw tokeniser.Exception($"NextToken(): unexpected character '{(char)b}'.");
}
//we come here when array is empty but has some whitespace
}
}
tokeniser.GetNextByte();
}
public void RecognisesIntegerConstants()
{
using var t = new Tokeniser("12345");
Token token = t.GetNextToken();
token.Type.Should().Be(NodeType.IntegerConstant);
token.Value.Should().Be("12345");
}
public void RecognisesStringConstants()
{
using var t = new Tokeniser("\"the quick brown fox, jumps over 23 ______ (of something)\"");
Token token = t.GetNextToken();
token.Type.Should().Be(NodeType.StringConstant);
token.Value.Should().Be("the quick brown fox, jumps over 23 ______ (of something)");
}
public void RecognisesStraightforwardIdentifiers()
{
using var t = new Tokeniser("counter");
Token token = t.GetNextToken();
token.Type.Should().Be(NodeType.Identifier);
token.Value.Should().Be("counter");
}
public void RecognisesMoreComplexIdentifiers()
{
using var t = new Tokeniser("first_3_entries");
Token token = t.GetNextToken();
token.Type.Should().Be(NodeType.Identifier);
token.Value.Should().Be("first_3_entries");
}
public void IgnoresBlockCommentsWithinALine()
{
using var t = new Tokeniser("/* get outta here */ return");
t.GetNextToken().Value.Should().Be("return");
t.GetNextToken().Should().BeNull();
}
public void IsImmediatelyAtEndIfInputIsAllWhitespace()
{
using var t = new Tokeniser(" \n \t \r\n ");
t.GetNextToken().Should().BeNull();
}
public void IsImmediatelyAtEndIfInputIsEmpty()
{
using var t = new Tokeniser("");
t.GetNextToken().Should().BeNull();
}
public void IgnoresWhitespaceWhenCheckingIfAtEnd()
{
using var t = new Tokeniser("class\n\t\t \n ");
t.GetNextToken();
t.GetNextToken().Should().BeNull();
}
public void NextTokenIsNullIfAtEnd()
{
using var t = new Tokeniser("class");
t.GetNextToken();
t.GetNextToken().Should().BeNull();
}
public void RecognisesAKeywordToken()
{
using var t = new Tokeniser("class");
t.GetNextToken().Type.Should().Be(NodeType.Keyword);
}
public void HandlesCombinationsOfTypes()
{
using var t = new Tokeniser("let first_3_entries=\"three\";let x=x+1;\nreturn x;\n");
Token token = t.GetNextToken();
token.Value.Should().Be("let");
token.Type.Should().Be(NodeType.Keyword);
token = t.GetNextToken();
token.Value.Should().Be("first_3_entries");
token.Type.Should().Be(NodeType.Identifier);
token = t.GetNextToken();
token.Value.Should().Be("=");
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
token.Value.Should().Be("three");
token.Type.Should().Be(NodeType.StringConstant);
token = t.GetNextToken();
token.Value.Should().Be(";");
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
token.Value.Should().Be("let");
token.Type.Should().Be(NodeType.Keyword);
token = t.GetNextToken();
token.Value.Should().Be("x");
token.Type.Should().Be(NodeType.Identifier);
token = t.GetNextToken();
token.Value.Should().Be("=");
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
token.Value.Should().Be("x");
token.Type.Should().Be(NodeType.Identifier);
token = t.GetNextToken();
token.Value.Should().Be("+");
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
token.Value.Should().Be("1");
token.Type.Should().Be(NodeType.IntegerConstant);
token = t.GetNextToken();
token.Value.Should().Be(";");
token.Type.Should().Be(NodeType.Symbol);
token = t.GetNextToken();
token.Value.Should().Be("return");
token.Type.Should().Be(NodeType.Keyword);
token = t.GetNextToken();
token.Value.Should().Be("x");
token.Type.Should().Be(NodeType.Identifier);
token = t.GetNextToken();
token.Value.Should().Be(";");
token.Type.Should().Be(NodeType.Symbol);
t.GetNextToken().Should().BeNull();
}
public void IgnoresBlockCommentsAcrossMultipleLines()
{
using var t = new Tokeniser("/* get outta\nhere */\nreturn");
t.GetNextToken().Value.Should().Be("return");
t.GetNextToken().Should().BeNull();
}
public void IgnoresLineComments()
{
using var t = new Tokeniser("// this is a comment\nreturn");
t.GetNextToken().Value.Should().Be("return");
t.GetNextToken().Should().BeNull();
}
public void RecognisesTheClassKeyword()
{
using var t = new Tokeniser("class");
t.GetNextToken().Value.Should().Be("class");
}
public void TestTokeniser()
{
var expectedStrings = new List <string>();
var testString = "false" + Environment.NewLine;
expectedStrings.Add("False");
testString += "true false" + Environment.NewLine;
expectedStrings.Add("True");
expectedStrings.Add("False");
testString += "1 -2 00 +987654321" + Environment.NewLine;
expectedStrings.Add("1");
expectedStrings.Add("-2");
expectedStrings.Add("0");
expectedStrings.Add("987654321");
testString += "+123.4 34.5 34. +.2 .1 0.0 -.002 -3.62" + Environment.NewLine;
expectedStrings.Add("123.4");
expectedStrings.Add("34.5");
expectedStrings.Add("34");
expectedStrings.Add("0.2");
expectedStrings.Add("0.1");
expectedStrings.Add("0");
expectedStrings.Add("-0.002");
expectedStrings.Add("-3.62");
testString += "/name /n " + Environment.NewLine;
expectedStrings.Add("/name");
expectedStrings.Add("/n");
testString += "(string)<112233445566778899AABCCDDEEFF>()<>" + Environment.NewLine;
expectedStrings.Add("\"string\"");
expectedStrings.Add("\"<112233445566778899AABCCDDEEFF>\"");
expectedStrings.Add("\"\"");
expectedStrings.Add("\"<>\"");
testString += " (string) <112233445566778899AABCCDDEEFF> ( ) < > " + Environment.NewLine;
expectedStrings.Add("\"string\"");
expectedStrings.Add("\"<112233445566778899AABCCDDEEFF>\"");
expectedStrings.Add("\" \"");
expectedStrings.Add("\"< >\"");
testString += "(a string can be\r\n on 2 lines or more) (a string can contain ()matched brackets)" + Environment.NewLine;
expectedStrings.Add("\"a string can be\r\n on 2 lines or more\"");
expectedStrings.Add("\"a string can contain ()matched brackets\"");
testString += "(a string with one open \\( bracket) (a string with one closing \\) bracket)" + Environment.NewLine;
expectedStrings.Add("\"a string with one open \\( bracket\"");
expectedStrings.Add("\"a string with one closing \\) bracket\"");
testString += "1%comment\n2" + Environment.NewLine;
expectedStrings.Add("1");
expectedStrings.Add("2");
testString += "1 %comment \n%comment\n % comment \n 2" + Environment.NewLine;
expectedStrings.Add("1");
expectedStrings.Add("2");
testString += " [ /someName false -0 (string) ] [ [ (array in array) ] true ] " + Environment.NewLine;
expectedStrings.Add("[/someName False 0 \"string\"]\r\n");
expectedStrings.Add("[\r\n[\"array in array\"]\r\n True]\r\n");
testString += "[/someName false -0(string)][[(array in array)]true]" + Environment.NewLine;
expectedStrings.Add("[/someName False 0 \"string\"]\r\n");
expectedStrings.Add("[\r\n[\"array in array\"]\r\n True]\r\n");
testString += "[/someName%\n]" + Environment.NewLine;
expectedStrings.Add("[/someName]\r\n");
testString += "[ /someName % comment \n /anotherName]" + Environment.NewLine;
expectedStrings.Add("[/someName /anotherName]\r\n");
testString += " << /Name1 123 >> " + Environment.NewLine;
expectedStrings.Add(">>\r\n /Name1 123\r\n<<\r\n");
testString += "<</Name1 124>>" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Name1 124\r\n<<\r\n");
testString += "<< /Name1 125 /Name2 [ (string) (array) 126 ] /Name3 << /subName1 127 /subName2 true >> /Name4 (another string) /Name5 <112233EE> >>" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Name1 125\r\n /Name2 [\"string\" \"array\" 126]\r\n\r\n /Name3 >>\r\n /subName1 127\r\n /subName2 True\r\n<<\r\n\r\n /Name4 \"another string\"\r\n /Name5 \"<112233EE>\"\r\n<<\r\n");
testString += "<</Name1 223/Name2[(string)(array)224]/Name3<</subName1 225/subName2 true>>/Name4(another string)/Name5<222233EE>>>" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Name1 223\r\n /Name2 [\"string\" \"array\" 224]\r\n\r\n /Name3 >>\r\n /subName1 225\r\n /subName2 True\r\n<<\r\n\r\n /Name4 \"another string\"\r\n /Name5 \"<222233EE>\"\r\n<<\r\n");
testString += "<</Length 17>>\r\nstream\r\n01234567890123456\r\nendstream\r\n" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Length 17\r\n<<\r\nstream 879, 17 endstream\r\n");
testString += "<< /Length 17 /Filter [ /FlateDecode ] >>\r\nstream\r\nx\x009Ck`\x0000\x0002\x0009\x00DE\x0003\x000C\x00B8\x0000\x0000( \x0001f\r\nendstream\r\n" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Length 17\r\n /Filter [/FlateDecode]\r\n\r\n<<\r\nstream 962, 17 endstream\r\n");
testString += "null" + Environment.NewLine;
expectedStrings.Add("null");
testString += " 1 0 obj \r\n(a string) \r\nendobj" + Environment.NewLine;
expectedStrings.Add("\"a string\" obj 1 0");
testString += "2 1 obj(a string2)endobj" + Environment.NewLine;
expectedStrings.Add("\"a string2\" obj 2 1");
testString += "3 2 R" + Environment.NewLine;
expectedStrings.Add("Ref obj 3 2");
testString += "7 0 obj <</Length 8 0 R>>stream\n12345678\nendstream endobj 8 0 obj 9 endobj" + Environment.NewLine;
expectedStrings.Add(">>\r\n /Length ref 8 0\r\n<<\r\nstream 1101, 8 endstream obj 7 0\r\n");
expectedStrings.Add("9 obj 8 0");
var testStringuffer = new byte[testString.Length];
for (int i = 0; i < testString.Length; i++)
{
testStringuffer[i] = (byte)testString[i];
}
var tokeniser = new Tokeniser(testStringuffer);
foreach (var expectedString in expectedStrings)
{
Assert.AreEqual(expectedString, tokeniser.GetNextToken().ToString());
}
}
