//=========================================================================================
internal override ScannerSpecification CreateScannerSpecification()
{
var oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_', '@');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("minus", '-');
oSpec.AddLiteral("asterisk", '*');
oSpec.AddLiteral("slash", '/');
oSpec.AddLiteral("poundkey", '#');
oSpec.AddLiteral("operators", '=', '>', '<', ';', ',', '.', '+', ')', '(', '|', '&');
oSpec.AddLiteral("n", 'N', 'n');
oSpec.AddLiteral("singleQuote", '\'');
oSpec.AddLiteral("doubleQuote", '"');
oSpec.AddLiteral("caretReturn", '\n');
oSpec.AddLiteral("tilda", '`');
oSpec.AddTokenDeclaration("id", "(n|l){l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("operator", "(operators|minus)"); // |asterisk
oSpec.AddBoundedToken("stringConst", "singleQuote", "singleQuote", "singleQuote");
oSpec.AddBoundedToken("stringConst", "n singleQuote", "singleQuote", "singleQuote");
oSpec.AddBoundedToken("stringConst", "doubleQuote", "doubleQuote", "doubleQuote");
oSpec.AddBoundedToken("comment1", "slash asterisk", "asterisk slash", null);
oSpec.AddBoundedToken("comment2", "minus minus", "caretReturn", null);
oSpec.AddBoundedToken("comment3", "poundkey", "caretReturn", null);
oSpec.AddBoundedToken("quotedId", "tilda", "tilda", "tilda");
return oSpec;
}
//=========================================================================================
internal override ScannerSpecification CreateScannerSpecification()
{
var oSpec = new ScannerSpecification();
oSpec.AddLiteral("letter", CharType.Letters, '_');
oSpec.AddLiteral("digit", CharType.Numbers);
oSpec.AddLiteral("lt", '<');
oSpec.AddLiteral("gt", '>');
oSpec.AddLiteral("sqBr1", '[');
oSpec.AddLiteral("sqBr2", ']');
oSpec.AddLiteral("slash", '/');
oSpec.AddLiteral("quest", '?');
oSpec.AddLiteral("excl", '!');
oSpec.AddLiteral("minus", '-');
oSpec.AddLiteral("eq", '=');
oSpec.AddLiteral("doubleQuote", '"');
oSpec.AddLiteral("c", 'C');
oSpec.AddLiteral("d", 'D');
oSpec.AddLiteral("a", 'A');
oSpec.AddLiteral("t", 'T');
oSpec.AddTokenDeclaration("id", "letter{letter|digit}");
oSpec.AddTokenDeclaration("lt", "lt[slash|quest]");
oSpec.AddTokenDeclaration("gt", "[slash|quest]gt");
oSpec.AddTokenDeclaration("eq", "eq");
oSpec.AddBoundedToken("attrValue", "doubleQuote", "doubleQuote", null, 1, 1);
oSpec.AddBoundedToken("comment", "lt excl minus minus", "minus minus gt", null, 4, 3);
oSpec.AddBoundedToken("cdata", "lt excl sqBr1 c d a t a sqBr1", "sqBr2 sqBr2 gt", null, 9, 3);
return oSpec;
}
public void MultiLineTokens()
{
ScannerSpecification oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("br1", '[');
oSpec.AddLiteral("br2", ']');
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddBoundedToken("id2", "br1", "br2", null);
StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4));
oStateParser.Spec.AddRule("id", "id|id2");
{
oStateParser.SetSource(
@"select [
x int] from x
");
this.ReadAndAssertToken(oStateParser, "id", "select");
this.ReadAndAssertToken(oStateParser, "id2", "[");
this.ReadAndAssertToken(oStateParser, "id2", " x int]");
}
}
//=========================================================================================
internal override ScannerSpecification CreateScannerSpecification()
{
var oSpec = new ScannerSpecification();
oSpec.AddLiteral("any", CharType.AnyNonSpaces);
oSpec.AddTokenDeclaration("all", "any{any}");
return oSpec;
}
//=========================================================================================
public StateScanner(ScannerSpecification specification, int tabsize)
{
this.Errors = new List <string>();
this.Specification = specification;
this._Reader = new BufferReader(tabsize);
this.TabSize = tabsize;
}
//=========================================================================================
public StateScanner(ScannerSpecification specification, int tabsize)
{
this.Errors = new List<string>();
this.Specification = specification;
this._Reader = new BufferReader(tabsize);
this.TabSize = tabsize;
}
//=========================================================================================
void IsInvalidToken(ScannerSpecification spec, string text)
{
StateScanner oStateScanner = new StateScanner(spec, 4);
oStateScanner.SetSource(text + "|");
oStateScanner.ReadNextToken();
Assert.That(oStateScanner.Errors, Has.Count.EqualTo(1));
}
//=========================================================================================
internal override ParserSpecification CreateParserSpecification(ScannerSpecification scannerSpecification)
{
var oSpec = new ParserSpecification();
oSpec.AddRule("Tag", "lt?separator? id?tagName? {id?attribute? eq?separator? attrValue?attributeValue?} gt?separator?");
oSpec.AddRule("Comment", "comment?comments?");
oSpec.AddRule("Cdata", "cdata?cdata?");
return oSpec;
}
//=========================================================================================
public StateParser(SyntaxSettings settings, int tabsize)
: this()
{
this.Settings = settings;
ScannerSpecification oScannerSpec = this.Settings.CreateScannerSpecification();
this.Scanner = new StateScanner(oScannerSpec, tabsize);
this.Spec = this.Settings.CreateParserSpecification(oScannerSpec);
}
//=========================================================================================
ScannerSpecification CreateSpecification()
{
ScannerSpecification oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("minus", '-');
oSpec.AddLiteral("plus", '+');
oSpec.AddLiteral("x", '|', '=', ';');
oSpec.AddLiteral("slash", '/');
oSpec.AddLiteral("asterisk", '*');
oSpec.AddLiteral("n", 'N');
oSpec.AddLiteral("singleQuote", '\'');
oSpec.AddLiteral("doubleQuote", '"');
oSpec.AddLiteral("backSlash", '\\');
oSpec.AddLiteral("caretReturn", '\n');
return oSpec;
}
//=========================================================================================
internal override ScannerSpecification CreateScannerSpecification()
{
var oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("minus", '-');
oSpec.AddLiteral("asterisk", '*');
oSpec.AddLiteral("slash", '/');
oSpec.AddLiteral("backSlash", '\\');
oSpec.AddLiteral("operators", '=', '>', '<', ';', ',', '.', '+', ')', '(', '|', '&');
oSpec.AddLiteral("singleQuote", '\'');
oSpec.AddLiteral("doubleQuote", '"');
oSpec.AddLiteral("caretReturn", '\n');
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("operator", "(operators|minus)");
oSpec.AddBoundedToken("charConst", "singleQuote", "singleQuote", "backSlash");
oSpec.AddBoundedToken("comment1", "slash asterisk", "asterisk slash", null);
oSpec.AddBoundedToken("comment2", "slash slash", "caretReturn", null);
oSpec.AddBoundedToken("stringConst", "doubleQuote", "doubleQuote", "backSlash");
return oSpec;
}
public void VariablesAssignment()
{
ScannerSpecification oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("x", '=', ';');
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("sep", "x");
StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4));
oStateParser.Spec.AddRule("Tag", "id sep number sep");
{
oStateParser.SetSource("sText=10;");
this.ReadAndAssertToken(oStateParser, "id", "sText");
this.ReadAndAssertToken(oStateParser, "sep", "=");
this.ReadAndAssertToken(oStateParser, "number", "10");
this.ReadAndAssertToken(oStateParser, "sep", ";");
var oToken = oStateParser.GetNextToken();
Assert.IsNull(oToken);
}
}
//=========================================================================================
internal virtual ParserSpecification CreateParserSpecification(ScannerSpecification scannerSpecification)
{
///Создание спецификации для парсера ПО УМОЛЧАНИЮ.
///По умолчанию все правила состоят из одного токена.
var oSpec = new ParserSpecification();
Dictionary<string, string> ruleNames = new Dictionary<string, string>();
///Найдем все упоминающиеся имена токенов и включим
foreach (var oState in scannerSpecification.States)
{
string sTokenName = oState.ResultTokenName;
if (!string.IsNullOrEmpty(sTokenName) && !ruleNames.ContainsKey(sTokenName))
{
oSpec.AddRule(sTokenName, sTokenName);
ruleNames.Add(sTokenName, sTokenName);
}
}
return oSpec;
}
