public void Should_Parse_Right_Parenthesis_Correctly()
{
// Given
var tokenizer = new ScriptTokenizer(" ) ");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.RightParenthesis, token.Type);
Assert.Equal(1, token.Index);
Assert.Equal(1, token.Length);
}
public void Should_Parse_Left_Brace_Correctly()
{
// Given
var tokenizer = new ScriptTokenizer(" { ");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.LeftBrace, token.Type);
Assert.Equal(1, token.Index);
Assert.Equal(1, token.Length);
}
[InlineData("\"Hello \\\"World\\\"\" ", 0, 17)] // "Hello \"World\""
public void Should_Parse_Strings_Correctly(string content, int start, int end)
{
// Given
var tokenizer = new ScriptTokenizer(content);
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.String, token.Type);
Assert.Equal(start, token.Index);
Assert.Equal(end, token.Length);
}
public void Should_Skip_New_Line()
{
// Given
var tokenizer = new ScriptTokenizer(" \n Hello ");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.Word, token.Type);
Assert.Equal(3, token.Index);
Assert.Equal(5, token.Length);
}
public void Should_Skip_Carriage_Return()
{
// Given
var tokenizer = new ScriptTokenizer(" \r\n Hello ");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.Word, token.Type);
Assert.Equal(4, token.Index);
Assert.Equal(5, token.Length);
}
public void Should_Skip_Single_Line_Comment()
{
// Given
var tokenizer = new ScriptTokenizer("// This should be ignored \n {");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.LeftBrace, token.Type);
Assert.Equal(28, token.Index);
Assert.Equal(1, token.Length);
}
public void Should_Skip_Multi_Line_Comment()
{
// Given
var tokenizer = new ScriptTokenizer("/* This should be ignored \n And this as well */ {");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.LeftBrace, token.Type);
Assert.Equal(50, token.Index);
Assert.Equal(1, token.Length);
}
public void Should_Parse_Character_Correctly(string content, int index, int length)
{
// Given
var tokenizer = new ScriptTokenizer(content);
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.Character, token.Type);
Assert.Equal(index, token.Index);
Assert.Equal(length, token.Length);
}
public void Should_Parse_Semicolon_Correctly()
{
// Given
var tokenizer = new ScriptTokenizer(" ; ");
// When
var token = tokenizer.GetNextToken();
// Then
Assert.Equal(ScriptTokenType.Semicolon, token.Type);
Assert.Equal(1, token.Index);
Assert.Equal(1, token.Length);
}
//////////////////////////////////////////////////////////////////////////
private void ScanScript(string ScriptFile)
{
try
{
FileStream fs = new FileStream(ScriptFile, FileMode.Open, FileAccess.Read, FileShare.Read);
ScriptTokenizer Tokenizer = new ScriptTokenizer(fs, false);
Tokenizer.GetTokens();
for (int i = 0; i < Tokenizer.Tokens.Count; i++)
{
ScriptToken PrevToken = i > 0 ? Tokenizer.Tokens[i - 1] : null;
ScriptToken Token = Tokenizer.Tokens[i];
// some quick checks
if (!Token.IsString)
{
continue;
}
if (PrevToken != null)
{
if ((PrevToken.IsKeyword && PrevToken.Value == "on") ||
(PrevToken.IsKeyword && PrevToken.Value == "extern") ||
(PrevToken.IsIdentifier && PrevToken.Value == "#include"))
{
continue;
}
}
string Str = Token.Value.Substring(1, Token.Value.Length - 2);
if (Str == string.Empty)
{
continue;
}
Strings.Add(new StringLocation(Str, ScriptFile, Token.StartLine, StringLocation.StringType.ScriptFile));
}
}
catch
{
AddLog(LogSeverity.Error, "Error parsing script '" + ScriptFile + "'");
}
}
static T TestParse <T>(string source, ICfgNode expected, TokenListParser <ScriptToken, T> parser) where T : ICfgNode
{
var tokens = ScriptTokenizer.Tokenize(source);
if (!tokens.HasValue)
{
throw new InvalidOperationException(
$"Tokenization failure: {tokens.ErrorMessage} at {tokens.ErrorPosition}");
}
var filtered = ScriptParser.FilterTokens(tokens.Value);
var parsed = parser.TryParse(filtered);
if (!parsed.HasValue)
{
throw new InvalidOperationException($"Parse failure: {parsed}");
}
Assert.Equal(expected, parsed.Value);
return(parsed.Value);
}
public bool Parse()
{
if (mScript == null)
{
return(false);
}
if (mScript.Count < 1)
{
mScript = null; return(false);
}
if (mScript.Next() == false)
{
mScript = null; return(false);
}
var head = mScript.Head;
try {
mChain.Invoke(mArgs);
} catch (System.Exception e) {
throw new ScriptUnparsable(head, mScript.Line, e);
}
return(true);
}
public void SetScript(string str)
{
mScript = ScriptTokenizer.Load(str);
mArgs.Value = mScript;
}
public void Init(string src, string path)
{
TokenizerBase tkz = new ScriptTokenizer();
LineBufferedReader r = LineBufferedReader.GetInstanceWithText(src, path);
tkz.Init(r);
Init(tkz);
}
