//========================================================================================= internal override ScannerSpecification CreateScannerSpecification() { var oSpec = new ScannerSpecification(); oSpec.AddLiteral("l", CharType.Letters, '_', '@'); oSpec.AddLiteral("d", CharType.Numbers); oSpec.AddLiteral("minus", '-'); oSpec.AddLiteral("asterisk", '*'); oSpec.AddLiteral("slash", '/'); oSpec.AddLiteral("poundkey", '#'); oSpec.AddLiteral("operators", '=', '>', '<', ';', ',', '.', '+', ')', '(', '|', '&'); oSpec.AddLiteral("n", 'N', 'n'); oSpec.AddLiteral("singleQuote", '\''); oSpec.AddLiteral("doubleQuote", '"'); oSpec.AddLiteral("caretReturn", '\n'); oSpec.AddLiteral("tilda", '`'); oSpec.AddTokenDeclaration("id", "(n|l){l|d}"); oSpec.AddTokenDeclaration("number", "d{d}"); oSpec.AddTokenDeclaration("operator", "(operators|minus)"); // |asterisk oSpec.AddBoundedToken("stringConst", "singleQuote", "singleQuote", "singleQuote"); oSpec.AddBoundedToken("stringConst", "n singleQuote", "singleQuote", "singleQuote"); oSpec.AddBoundedToken("stringConst", "doubleQuote", "doubleQuote", "doubleQuote"); oSpec.AddBoundedToken("comment1", "slash asterisk", "asterisk slash", null); oSpec.AddBoundedToken("comment2", "minus minus", "caretReturn", null); oSpec.AddBoundedToken("comment3", "poundkey", "caretReturn", null); oSpec.AddBoundedToken("quotedId", "tilda", "tilda", "tilda"); return oSpec; }
//========================================================================================= internal override ScannerSpecification CreateScannerSpecification() { var oSpec = new ScannerSpecification(); oSpec.AddLiteral("letter", CharType.Letters, '_'); oSpec.AddLiteral("digit", CharType.Numbers); oSpec.AddLiteral("lt", '<'); oSpec.AddLiteral("gt", '>'); oSpec.AddLiteral("sqBr1", '['); oSpec.AddLiteral("sqBr2", ']'); oSpec.AddLiteral("slash", '/'); oSpec.AddLiteral("quest", '?'); oSpec.AddLiteral("excl", '!'); oSpec.AddLiteral("minus", '-'); oSpec.AddLiteral("eq", '='); oSpec.AddLiteral("doubleQuote", '"'); oSpec.AddLiteral("c", 'C'); oSpec.AddLiteral("d", 'D'); oSpec.AddLiteral("a", 'A'); oSpec.AddLiteral("t", 'T'); oSpec.AddTokenDeclaration("id", "letter{letter|digit}"); oSpec.AddTokenDeclaration("lt", "lt[slash|quest]"); oSpec.AddTokenDeclaration("gt", "[slash|quest]gt"); oSpec.AddTokenDeclaration("eq", "eq"); oSpec.AddBoundedToken("attrValue", "doubleQuote", "doubleQuote", null, 1, 1); oSpec.AddBoundedToken("comment", "lt excl minus minus", "minus minus gt", null, 4, 3); oSpec.AddBoundedToken("cdata", "lt excl sqBr1 c d a t a sqBr1", "sqBr2 sqBr2 gt", null, 9, 3); return oSpec; }
public void MultiLineTokens() { ScannerSpecification oSpec = new ScannerSpecification(); oSpec.AddLiteral("l", CharType.Letters, '_'); oSpec.AddLiteral("d", CharType.Numbers); oSpec.AddLiteral("br1", '['); oSpec.AddLiteral("br2", ']'); oSpec.AddTokenDeclaration("id", "l{l|d}"); oSpec.AddBoundedToken("id2", "br1", "br2", null); StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4)); oStateParser.Spec.AddRule("id", "id|id2"); { oStateParser.SetSource( @"select [ x int] from x "); this.ReadAndAssertToken(oStateParser, "id", "select"); this.ReadAndAssertToken(oStateParser, "id2", "["); this.ReadAndAssertToken(oStateParser, "id2", " x int]"); } }
//========================================================================================= internal override ScannerSpecification CreateScannerSpecification() { var oSpec = new ScannerSpecification(); oSpec.AddLiteral("any", CharType.AnyNonSpaces); oSpec.AddTokenDeclaration("all", "any{any}"); return oSpec; }
//========================================================================================= public StateScanner(ScannerSpecification specification, int tabsize) { this.Errors = new List <string>(); this.Specification = specification; this._Reader = new BufferReader(tabsize); this.TabSize = tabsize; }
//========================================================================================= public StateScanner(ScannerSpecification specification, int tabsize) { this.Errors = new List<string>(); this.Specification = specification; this._Reader = new BufferReader(tabsize); this.TabSize = tabsize; }
//========================================================================================= void IsInvalidToken(ScannerSpecification spec, string text) { StateScanner oStateScanner = new StateScanner(spec, 4); oStateScanner.SetSource(text + "|"); oStateScanner.ReadNextToken(); Assert.That(oStateScanner.Errors, Has.Count.EqualTo(1)); }
//========================================================================================= internal override ParserSpecification CreateParserSpecification(ScannerSpecification scannerSpecification) { var oSpec = new ParserSpecification(); oSpec.AddRule("Tag", "lt?separator? id?tagName? {id?attribute? eq?separator? attrValue?attributeValue?} gt?separator?"); oSpec.AddRule("Comment", "comment?comments?"); oSpec.AddRule("Cdata", "cdata?cdata?"); return oSpec; }
//========================================================================================= public StateParser(SyntaxSettings settings, int tabsize) : this() { this.Settings = settings; ScannerSpecification oScannerSpec = this.Settings.CreateScannerSpecification(); this.Scanner = new StateScanner(oScannerSpec, tabsize); this.Spec = this.Settings.CreateParserSpecification(oScannerSpec); }
//========================================================================================= ScannerSpecification CreateSpecification() { ScannerSpecification oSpec = new ScannerSpecification(); oSpec.AddLiteral("l", CharType.Letters, '_'); oSpec.AddLiteral("d", CharType.Numbers); oSpec.AddLiteral("minus", '-'); oSpec.AddLiteral("plus", '+'); oSpec.AddLiteral("x", '|', '=', ';'); oSpec.AddLiteral("slash", '/'); oSpec.AddLiteral("asterisk", '*'); oSpec.AddLiteral("n", 'N'); oSpec.AddLiteral("singleQuote", '\''); oSpec.AddLiteral("doubleQuote", '"'); oSpec.AddLiteral("backSlash", '\\'); oSpec.AddLiteral("caretReturn", '\n'); return oSpec; }
//========================================================================================= internal override ScannerSpecification CreateScannerSpecification() { var oSpec = new ScannerSpecification(); oSpec.AddLiteral("l", CharType.Letters, '_'); oSpec.AddLiteral("d", CharType.Numbers); oSpec.AddLiteral("minus", '-'); oSpec.AddLiteral("asterisk", '*'); oSpec.AddLiteral("slash", '/'); oSpec.AddLiteral("backSlash", '\\'); oSpec.AddLiteral("operators", '=', '>', '<', ';', ',', '.', '+', ')', '(', '|', '&'); oSpec.AddLiteral("singleQuote", '\''); oSpec.AddLiteral("doubleQuote", '"'); oSpec.AddLiteral("caretReturn", '\n'); oSpec.AddTokenDeclaration("id", "l{l|d}"); oSpec.AddTokenDeclaration("number", "d{d}"); oSpec.AddTokenDeclaration("operator", "(operators|minus)"); oSpec.AddBoundedToken("charConst", "singleQuote", "singleQuote", "backSlash"); oSpec.AddBoundedToken("comment1", "slash asterisk", "asterisk slash", null); oSpec.AddBoundedToken("comment2", "slash slash", "caretReturn", null); oSpec.AddBoundedToken("stringConst", "doubleQuote", "doubleQuote", "backSlash"); return oSpec; }
public void VariablesAssignment() { ScannerSpecification oSpec = new ScannerSpecification(); oSpec.AddLiteral("l", CharType.Letters, '_'); oSpec.AddLiteral("d", CharType.Numbers); oSpec.AddLiteral("x", '=', ';'); oSpec.AddTokenDeclaration("id", "l{l|d}"); oSpec.AddTokenDeclaration("number", "d{d}"); oSpec.AddTokenDeclaration("sep", "x"); StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4)); oStateParser.Spec.AddRule("Tag", "id sep number sep"); { oStateParser.SetSource("sText=10;"); this.ReadAndAssertToken(oStateParser, "id", "sText"); this.ReadAndAssertToken(oStateParser, "sep", "="); this.ReadAndAssertToken(oStateParser, "number", "10"); this.ReadAndAssertToken(oStateParser, "sep", ";"); var oToken = oStateParser.GetNextToken(); Assert.IsNull(oToken); } }
//========================================================================================= internal virtual ParserSpecification CreateParserSpecification(ScannerSpecification scannerSpecification) { ///Создание спецификации для парсера ПО УМОЛЧАНИЮ. ///По умолчанию все правила состоят из одного токена. var oSpec = new ParserSpecification(); Dictionary<string, string> ruleNames = new Dictionary<string, string>(); ///Найдем все упоминающиеся имена токенов и включим foreach (var oState in scannerSpecification.States) { string sTokenName = oState.ResultTokenName; if (!string.IsNullOrEmpty(sTokenName) && !ruleNames.ContainsKey(sTokenName)) { oSpec.AddRule(sTokenName, sTokenName); ruleNames.Add(sTokenName, sTokenName); } } return oSpec; }