public void ReadAllTokensWithSpaces()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("sep", "x");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource(" q2 \t 33 ");
//Пропускаем начальный пробел
var oToken = this.ReadAndAssertToken(oStateScanner, "id", "q2");
Assert.That(oToken.Start.ToString(), Is.EqualTo("Ln:0, col:1, ch:1"));
Assert.That(oToken.End.ToString(), Is.EqualTo("Ln:0, col:3, ch:3"));
//Пропускаем пробелы и табы
oToken = this.ReadAndAssertToken(oStateScanner, "number", "33");
Assert.That(oToken.Start.ToString(), Is.EqualTo("Ln:0, col:9, ch:6"));
Assert.That(oToken.End.ToString(), Is.EqualTo("Ln:0, col:11, ch:8"));
oToken = oStateScanner.ReadNextToken();
Assert.IsNull(oToken);
}
//=========================================================================================
void IsInvalidToken(ScannerSpecification spec, string text)
{
StateScanner oStateScanner = new StateScanner(spec, 4);
oStateScanner.SetSource(text + "|");
oStateScanner.ReadNextToken();
Assert.That(oStateScanner.Errors, Has.Count.EqualTo(1));
}
//=========================================================================================
Token ReadAndAssertToken(StateScanner scanner, string tokenType, string text)
{
var oToken = scanner.ReadNextToken();
Assert.That(scanner.Errors, Has.Count.EqualTo(0));
Assert.That(oToken.Text, Is.EqualTo(text));
Assert.That(oToken.TokenTypeName, Is.EqualTo(tokenType));
return(oToken);
}
public void AlternativeOfSequences()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("s1", "l{plus plus|minus minus}d");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
this.IsValidToken(oSpec, "a2");
this.IsValidToken(oSpec, "a++2");
this.IsValidToken(oSpec, "b++++2");
this.IsValidToken(oSpec, "b++++++2");
this.IsInvalidToken(oSpec, "b+2");
}
public void ReadStringConstWithEscape_2()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddBoundedToken("s1", "singleQuote", "singleQuote", "singleQuote");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
const string sText = "'a '' 2 '''";
oStateScanner.SetSource(sText);
this.ReadAndAssertToken(oStateScanner, "s1", sText);
Assert.IsNull(oStateScanner.ReadNextToken());
}
public void VariablesAssignment()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("sep", "x");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource("sText=10;");
this.ReadAndAssertToken(oStateScanner, "id", "sText");
this.ReadAndAssertToken(oStateScanner, "sep", "=");
this.ReadAndAssertToken(oStateScanner, "number", "10");
this.ReadAndAssertToken(oStateScanner, "sep", ";");
}
public void ReadComment01()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddBoundedToken("s1", "slash asterisk", "asterisk slash", null);
StateScanner oStateScanner = new StateScanner(oSpec, 4);
const string sText = "/** x **/ select";
oStateScanner.SetSource(sText);
this.ReadAndAssertToken(oStateScanner, "s1", "/** x **/");
oStateScanner.ReadNextToken();
Assert.IsNull(oStateScanner.ReadNextToken());
}
public void CheckTokenRange_01()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("number", "d{d}");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource("33");
var oToken = this.ReadAndAssertToken(oStateScanner, "number", "33");
Assert.That(oToken.Start.ToString(), Is.EqualTo("Ln:0, col:0, ch:0"));
Assert.That(oToken.End.ToString(), Is.EqualTo("Ln:0, col:2, ch:2"));
oToken = oStateScanner.ReadNextToken();
Assert.IsNull(oToken);
}
public void ReadAllTokens()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("id", "l l");
oSpec.AddTokenDeclaration("number", "d d");
oSpec.AddTokenDeclaration("sep", "x");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource("qq|22");
this.ReadAndAssertToken(oStateScanner, "id", "qq");
this.ReadAndAssertToken(oStateScanner, "sep", "|");
this.ReadAndAssertToken(oStateScanner, "number", "22");
var oToken = oStateScanner.ReadNextToken();
Assert.IsNull(oToken);
}
public void OperatorsAndComment()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddBoundedToken("c1", "minus minus", "backSlash", null);
oSpec.AddBoundedToken("c2", "minus plus", "backSlash", null);
oSpec.AddTokenDeclaration("operations", "minus{d}");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
const string sText = "-+x\\ -23400 --+-+x\\";
oStateScanner.SetSource(sText);
this.ReadAndAssertToken(oStateScanner, "c2", "-+x\\");
this.ReadAndAssertToken(oStateScanner, "operations", "-23400");
this.ReadAndAssertToken(oStateScanner, "c1", "--+-+x\\");
oStateScanner.ReadNextToken();
Assert.IsNull(oStateScanner.ReadNextToken());
}
public void ReadSingleLineComments()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddBoundedToken("c1", "slash asterisk", "asterisk slash", null);
oSpec.AddBoundedToken("c2", "minus minus", "caretReturn", null);
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource("-- 22 33\r\n12");
{
var oToken = oStateScanner.ReadNextToken();
this.AssertTokenRange(oToken, 0, 0, 0, 1, 0, 0);
}
{
var oToken = oStateScanner.ReadNextToken();
this.AssertTokenRange(oToken, 1, 0, 0, 1, 2, 2);
}
}
public void CheckTokenRangeMultiLines()
{
ScannerSpecification oSpec = this.CreateSpecification();
oSpec.AddTokenDeclaration("number", "d{d}");
StateScanner oStateScanner = new StateScanner(oSpec, 4);
oStateScanner.SetSource("33\r\n2\n\n 3");
{
var oToken = oStateScanner.ReadNextToken();
this.AssertTokenRange(oToken, 0, 0, 0, 0, 2, 2);
}
{
var oToken = oStateScanner.ReadNextToken();
this.AssertTokenRange(oToken, 1, 0, 0, 1, 1, 1);
}
{
var oToken = oStateScanner.ReadNextToken();
this.AssertTokenRange(oToken, 3, 1, 1, 3, 2, 2);
}
}
