public void TestDollarAtStrings()
{
LocalisationParser parser = new LocalisationParser("TestDollarAtStrings.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("as\"\" { as\"d }d", parser.String);
Assert.AreEqual(TokenType.DollarAtString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("as\\\"\" { as\\\"d }d", parser.String);
Assert.AreEqual(TokenType.DollarAtString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("as\\\"\"{{ { a{}\r\ns\\\"d }d", parser.String);
Assert.AreEqual(TokenType.DollarAtString, parser.Type);
Assert.IsFalse(parser.NextString());
}
public void TestNewlines()
{
LocalisationParser parser = new LocalisationParser("a\nb\n\nc\n\n\nd\n\n", "");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(1, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(2, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(4, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(7, parser.Line);
Assert.IsFalse(parser.NextToken());
}
public void TestNoTokens()
{
LocalisationParser parser = new LocalisationParser(" - -- \n =+ = ", "");
Assert.IsFalse(parser.NextToken());
Assert.AreEqual(0, parser.Offset);
Assert.AreEqual(1, parser.Line);
Assert.AreEqual("- --", parser.Context);
}
public void TestEmptyString()
{
LocalisationParser parser = new LocalisationParser("", "");
Assert.AreEqual(0, parser.Offset);
Assert.AreEqual(1, parser.Line);
Assert.AreEqual("", parser.Context);
Assert.IsFalse(parser.NextToken());
}
public void TestNextString()
{
string path = Path.GetTempFileName();
File.WriteAllText(path, File.ReadAllText("TestNextString.txt"));
LocalisationParser parser = new LocalisationParser(path);
parser.Prefix = "p";
Assert.IsTrue(parser.NextString());
Assert.AreEqual("some string yay", parser.String);
Assert.AreEqual("foo", parser.Variable);
Assert.AreEqual("P_SomeStringYay", parser.FullKey);
parser.Key = "actually foo";
Assert.AreEqual("P_ActuallyFoo", parser.FullKey);
parser.ReplaceStringWithKey();
Assert.IsTrue(parser.NextString());
parser.Prefix = "no go";
Assert.IsTrue(parser.String.Length > 50);
Assert.AreEqual("foo_bar", parser.Variable);
Assert.AreEqual("No_go_FooBar", parser.FullKey);
parser.ReplaceStringWithKey();
Assert.IsTrue(parser.NextString());
parser.Prefix = "p!!";
Assert.AreEqual("good", parser.String);
Assert.AreEqual("Foo.bar", parser.Variable);
Assert.AreEqual("P_Good", parser.FullKey);
parser.ReplaceStringWithConstant();
Assert.IsTrue(parser.NextString());
Assert.AreEqual("bad", parser.String);
Assert.AreEqual("Foo.bar", parser.Variable);
Assert.AreEqual("P_Bad", parser.FullKey);
Assert.IsTrue(parser.NextString());
Assert.AreEqual("escapable", parser.Variable);
parser.ReplaceStringWithConstant();
Assert.IsTrue(parser.NextString());
Assert.AreEqual("unescapable", parser.Variable);
parser.ReplaceStringWithConstant();
Assert.IsFalse(parser.NextToken());
string[] expected = File.ReadAllLines("TestNextStringExpected.txt");
string[] actual = File.ReadAllLines(path);
Assert.AreEqual(expected.Length, actual.Length);
for (int i = 0; i < expected.Length; i++)
{
Assert.AreEqual(expected[i], actual[i]);
}
}
public void TestIdentifierTokens()
{
LocalisationParser parser = new LocalisationParser(" - This is.a \n Test.string a -- ok \n ", "");
string con1 = "- This is.a";
string con2 = "Test.string a -- ok";
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("This", parser.Token);
Assert.AreEqual(con1, parser.Context);
Assert.AreEqual(3, parser.Offset);
Assert.AreEqual(1, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("is.a", parser.Token);
Assert.AreEqual(con1, parser.Context);
Assert.AreEqual(8, parser.Offset);
Assert.AreEqual(1, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("Test.string", parser.Token);
Assert.AreEqual(con2, parser.Context);
Assert.AreEqual(15, parser.Offset);
Assert.AreEqual(2, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("a", parser.Token);
Assert.AreEqual(con2, parser.Context);
Assert.AreEqual(28, parser.Offset);
Assert.AreEqual(2, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("ok", parser.Token);
Assert.AreEqual(con2, parser.Context);
Assert.AreEqual(33, parser.Offset);
Assert.AreEqual(2, parser.Line);
Assert.IsFalse(parser.NextToken());
}
public void TestDollarStrings()
{
LocalisationParser parser = new LocalisationParser("TestDollarStrings.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\\\"", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("{asd}", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("one {asd} {another.poop} ok {asd}", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd{ }", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("sq}}{asdf\"{\"sdf} sdfiuhsdf { sadf }adf}}", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd{{", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd{{s}", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd{asd}", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd {{{{ ", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("asd {{{in\"si\"de}}} ", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(" { { \"\" } {{ \"\" }} } ", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("} {asd} asd {{", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(" {{ \\\"", parser.String);
Assert.AreEqual(TokenType.DollarString, parser.Type);
Assert.IsFalse(parser.NextString());
}
public void TestReplaceAll()
{
LocalisationParser parser = new LocalisationParser("TestReplaceAll.txt");
parser.ReplaceAll("doesn't work");
Assert.AreEqual("sweet sweet", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("sweet", parser.Token);
Assert.AreEqual(0, parser.Offset);
Assert.AreEqual(1, parser.Line);
parser.ReplaceAll("sour");
Assert.AreEqual("sour", parser.Token);
Assert.AreEqual(0, parser.Offset);
Assert.AreEqual(1, parser.Line);
Assert.AreEqual("sour sour", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("sour", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.String);
parser.ReplaceAll("-");
Assert.AreEqual("- in", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("in", parser.Token);
parser.ReplaceAll("not");
Assert.AreEqual("- not", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("not", parser.Token);
Assert.AreEqual("not the - not @\"two\"", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("the", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("not", parser.Token);
parser.ReplaceAll("foo");
Assert.AreEqual("not the - foo @\"two\"", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("long\r\n", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.Token);
Assert.AreEqual("\"two-two\"", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'k'", parser.Token);
parser.ReplaceAll("K");
Assert.AreEqual("K 'j'k'j'K", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'j'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("k", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'j'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("K", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("foo", parser.Token);
Assert.AreEqual("/* sweet */ foo", parser.Context);
Assert.IsFalse(parser.NextToken());
}
public void TestReplaceToken()
{
LocalisationParser parser = new LocalisationParser("TestReplaceToken.txt");
parser.ReplaceToken("doesn't work");
Assert.AreEqual("A bee C", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("bee", parser.Token);
Assert.AreEqual(2, parser.Offset);
parser.ReplaceToken("B");
Assert.AreEqual("B", parser.Token);
Assert.AreEqual(2, parser.Offset);
Assert.AreEqual("A B C", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("A B C", parser.Context);
Assert.AreEqual(1, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("ok", parser.String);
parser.ReplaceToken("nok");
Assert.AreEqual("nok", parser.Token);
Assert.AreEqual("nok // \"nok\"", parser.Context);
Assert.AreEqual(2, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"'\5'", parser.Token);
parser.ReplaceToken("'5'");
Assert.AreEqual("'5'", parser.Token);
Assert.AreEqual("'5' /* '5' */", parser.Context);
Assert.AreEqual(3, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(4, parser.Line);
Assert.AreEqual("@\"mul", parser.Context);
parser.ReplaceToken("YIP");
Assert.AreEqual(4, parser.Line);
Assert.AreEqual("YIPPEE", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(4, parser.Line);
Assert.AreEqual("PEE", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(5, parser.Line);
Assert.AreEqual("@\"not\"multi", parser.Context);
parser.ReplaceToken("DO\nIT\nGOOD");
Assert.AreEqual(5, parser.Line);
Assert.AreEqual("DO", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("multi", parser.Token);
Assert.AreEqual(7, parser.Line);
Assert.AreEqual("GOODmulti", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(8, parser.Line);
Assert.AreEqual("@\"", parser.Context);
parser.ReplaceToken("\n\n");
Assert.AreEqual(8, parser.Line);
Assert.AreEqual("", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("OK", parser.Token);
Assert.AreEqual(13, parser.Line);
Assert.IsFalse(parser.NextToken());
}
public void TestComments()
{
LocalisationParser parser = new LocalisationParser("TestComments.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("seen", parser.Token);
Assert.AreEqual(1, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("good", parser.String);
Assert.AreEqual(2, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(" // nope\r\n// ", parser.String);
Assert.AreEqual(3, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("a", parser.Token);
Assert.AreEqual(5, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("string", parser.String);
Assert.AreEqual(6, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("b", parser.Token);
Assert.AreEqual(7, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("A", parser.Token);
Assert.AreEqual(10, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("C", parser.Token);
Assert.AreEqual(10, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("1", parser.String);
Assert.AreEqual(11, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("2", parser.String);
Assert.AreEqual(11, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(" /* nope\r\n*/ ", parser.String);
Assert.AreEqual(17, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("Z", parser.Token);
Assert.AreEqual(23, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("X", parser.Token);
Assert.AreEqual(25, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("Y", parser.Token);
Assert.AreEqual(28, parser.Line);
Assert.IsFalse(parser.NextToken());
}
public void TestCharacters()
{
LocalisationParser parser = new LocalisationParser("TestCharacters.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'a'", parser.Token);
Assert.AreEqual(TokenType.Character, parser.Type);
Assert.AreEqual(0, parser.Offset);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(2, parser.Line);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'2'", parser.Token);
Assert.AreEqual(TokenType.Character, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Character, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'A'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'B'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"'\n'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"'\\'", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"'\''", parser.Token);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("'\"'", parser.Token);
Assert.AreEqual(TokenType.Character, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("string '", parser.String);
Assert.AreEqual(TokenType.String, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"'\\'", parser.Token);
Assert.AreEqual(TokenType.Character, parser.Type);
Assert.IsFalse(parser.NextToken());
}
public void TestAtStrings()
{
LocalisationParser parser = new LocalisationParser("TestAtStrings.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("@\"Some text\"", parser.Token);
Assert.AreEqual("Some text", parser.String);
Assert.AreEqual(TokenType.AtString, parser.Type);
Assert.AreEqual(0, parser.Offset);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(2, parser.Line);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.String);
Assert.AreEqual(TokenType.AtString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.AtString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("@\"\"", parser.Token);
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"\", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\"\"", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\"\" \"\" quote \"\"\"\" \"\"", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("multi\r\nline \"\"\r\n\r\nstring", parser.String);
Assert.AreEqual(8, parser.Line);
Assert.AreEqual("@\"multi", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.AreEqual(11, parser.Line);
Assert.AreEqual("string\"real", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.AreEqual(12, parser.Line);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\r\n", parser.String);
Assert.AreEqual(13, parser.Line);
Assert.AreEqual(TokenType.AtString, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\r\n\r\n", parser.String);
Assert.AreEqual(14, parser.Line);
Assert.AreEqual(TokenType.AtString, parser.Type);
Assert.AreEqual("\"@\"", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.AreEqual(18, parser.Line);
Assert.IsFalse(parser.NextToken());
}
public void TestStrings()
{
LocalisationParser parser = new LocalisationParser("TestStrings.txt");
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\"Some text\"", parser.Token);
Assert.AreEqual("Some text", parser.String);
Assert.AreEqual(TokenType.String, parser.Type);
Assert.AreEqual(0, parser.Offset);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(2, parser.Line);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("two", parser.String);
Assert.AreEqual(TokenType.String, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.String, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"cat\n", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"\\", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("\\\"", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(@"a\\\""\b\""c\""d\\", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("", parser.String);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.AreEqual("invalid", parser.Token);
Assert.AreEqual("\"invalid", parser.Context);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("still", parser.Token);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("bad", parser.Token);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("yes", parser.Token);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("no", parser.Token);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsTrue(parser.NextToken());
Assert.AreEqual("way", parser.Token);
Assert.AreEqual(TokenType.Identifier, parser.Type);
Assert.IsFalse(parser.NextToken());
}
