/// <summary> /// Initializes a new instance of the <c>JsonLexer</c> class. /// </summary> /// <param name="classificationTypeProvider">A <see cref="IJsonClassificationTypeProvider"/> that provides classification types used by this lexer.</param> public JsonLexer(IJsonClassificationTypeProvider classificationTypeProvider) { if ((classificationTypeProvider == null)) { throw new ArgumentNullException("classificationTypeProvider"); } // Create lexical states IKeyedObservableCollection <DynamicLexicalState> lexicalStates = ((IKeyedObservableCollection <DynamicLexicalState>)(this.LexicalStates)); this.DefaultLexicalState.Id = JsonLexicalStateId.Default; lexicalStates.Add(new DynamicLexicalState(JsonLexicalStateId.PrimaryString, "PrimaryString")); DynamicLexicalState lexicalState = null; DynamicLexicalScope lexicalScope = null; DynamicLexicalPatternGroup lexicalPatternGroup = null; // Initialize the Default lexical state lexicalState = lexicalStates["Default"]; lexicalState.ChildLexicalStates.Add(lexicalStates["PrimaryString"]); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Whitespace", null); lexicalPatternGroup.TokenId = JsonTokenId.Whitespace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{LineTerminatorWhitespace}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Comma", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.Comma; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(",")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Colon", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.Colon; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(":")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.OpenCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.CloseCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenSquareBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.OpenSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseSquareBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.CloseSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("]")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "True", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.True; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("true")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "False", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.False; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("false")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Null", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.Null; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("null")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Number", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = JsonTokenId.Number; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}* \\. {Digit}+ ([Ee] [\\+\\-]? {Digit}+)?")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}+ [Ee] [\\+\\-]? {Digit}+")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the PrimaryString lexical state lexicalState = lexicalStates["PrimaryString"]; lexicalState.DefaultClassificationType = classificationTypeProvider.String; lexicalState.DefaultTokenId = JsonTokenId.PrimaryStringText; lexicalState.DefaultTokenKey = "PrimaryStringText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "StringStartDelimiter", null); lexicalPatternGroup.TokenId = JsonTokenId.StringStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "StringEndDelimiter", null); lexicalPatternGroup.TokenId = JsonTokenId.StringEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "EscapedCharacter", null); lexicalPatternGroup.TokenId = JsonTokenId.EscapedCharacter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\"")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\\")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\/")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\b")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\f")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\n")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\r")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\t")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "EscapedUnicode", null); lexicalPatternGroup.TokenId = JsonTokenId.EscapedUnicode; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\\u[0-9a-fA-F]{4}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "StringText", null); lexicalPatternGroup.TokenId = JsonTokenId.StringText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[^\\\"\\\\\\n]+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); }
/// <summary> /// Initializes a new instance of the <c>PascalLexer</c> class. /// </summary> /// <param name="classificationTypeProvider">A <see cref="IPascalClassificationTypeProvider"/> that provides classification types used by this lexer.</param> public PascalLexer(IPascalClassificationTypeProvider classificationTypeProvider) { if ((classificationTypeProvider == null)) { throw new ArgumentNullException("classificationTypeProvider"); } // Create lexical macros this.LexicalMacros.Add(new DynamicLexicalMacro("Word", "[a-zA-Z0-9]")); this.LexicalMacros.Add(new DynamicLexicalMacro("NonWord", "[^a-zA-Z0-9]")); // Create lexical states IKeyedObservableCollection <DynamicLexicalState> lexicalStates = ((IKeyedObservableCollection <DynamicLexicalState>)(this.LexicalStates)); this.DefaultLexicalState.Id = PascalLexicalStateId.Default; lexicalStates.Add(new DynamicLexicalState(PascalLexicalStateId.PrimaryString, "PrimaryString")); lexicalStates.Add(new DynamicLexicalState(PascalLexicalStateId.MultiLineComment, "MultiLineComment")); DynamicLexicalState lexicalState = null; DynamicLexicalScope lexicalScope = null; DynamicLexicalPatternGroup lexicalPatternGroup = null; // Initialize the Default lexical state lexicalState = lexicalStates["Default"]; lexicalState.ChildLexicalStates.Add(lexicalStates["PrimaryString"]); lexicalState.ChildLexicalStates.Add(lexicalStates["MultiLineComment"]); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Whitespace", null); lexicalPatternGroup.TokenId = PascalTokenId.Whitespace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{Whitespace}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "LineTerminator", null); lexicalPatternGroup.TokenId = PascalTokenId.LineTerminator; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{LineTerminator}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenParenthesis", null); lexicalPatternGroup.TokenId = PascalTokenId.OpenParenthesis; lexicalPatternGroup.LookAheadPattern = "$|[^\\*\\.]"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("(")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseParenthesis", null); lexicalPatternGroup.TokenId = PascalTokenId.CloseParenthesis; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(")")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "OpenSquareBrace", null); lexicalPatternGroup.TokenId = PascalTokenId.OpenSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\[")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\(\\.")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "CloseSquareBrace", null); lexicalPatternGroup.TokenId = PascalTokenId.CloseSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\]")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\.\\)")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Punctuation", null); lexicalPatternGroup.TokenId = PascalTokenId.Punctuation; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(".. . , : ; ^ @")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Keyword", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = PascalTokenId.Keyword; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.Sensitive; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("and")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("array")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("begin")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("case")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("const")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("div")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("downto")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("do")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("else")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("end")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("file")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("for")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("function")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("goto")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("if")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("in")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("label")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("mod")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("nil")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("not")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("of")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("or")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("packed")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("procedure")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("program")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("record")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("repeat")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("set")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("then")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("to")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("type")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("until")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("var")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("while")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("with")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Identifier", classificationTypeProvider.Identifier); lexicalPatternGroup.TokenId = PascalTokenId.Identifier; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{Alpha}({Word})*")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Operator", classificationTypeProvider.Operator); lexicalPatternGroup.TokenId = PascalTokenId.Operator; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("*")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("+")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("-")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("/")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(":=")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("<=")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("<>")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("<")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("=")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(">=")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(">")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "RealNumber", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = PascalTokenId.RealNumber; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]? {Digit}+ \\. {Digit}+ (e [\\+\\-]? {Digit}+)?")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]? {Digit}+ e [\\+\\-]? {Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "IntegerNumber", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = PascalTokenId.IntegerNumber; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]? {Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the PrimaryString lexical state lexicalState = lexicalStates["PrimaryString"]; lexicalState.DefaultClassificationType = classificationTypeProvider.String; lexicalState.DefaultTokenId = PascalTokenId.PrimaryStringText; lexicalState.DefaultTokenKey = "PrimaryStringText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "PrimaryStringStartDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.PrimaryStringStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "PrimaryStringEndDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.PrimaryStringEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("((?<!\\\\)\\\"|\\n)")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "PrimaryStringEscapedDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.PrimaryStringEscapedDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"\"")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "PrimaryStringText", null); lexicalPatternGroup.TokenId = PascalTokenId.PrimaryStringText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("(\\\\\"|[^\"\\n])+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the MultiLineComment lexical state lexicalState = lexicalStates["MultiLineComment"]; lexicalState.DefaultClassificationType = classificationTypeProvider.Comment; lexicalState.DefaultTokenId = PascalTokenId.MultiLineCommentText; lexicalState.DefaultTokenKey = "MultiLineCommentText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentStartDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentEndDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("}")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentStartDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("(*")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentEndDelimiter", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("*)")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "MultiLineCommentLineTerminator", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentLineTerminator; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\n")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "MultiLineCommentText", null); lexicalPatternGroup.TokenId = PascalTokenId.MultiLineCommentText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[^\\}\\*\\n]+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); }
/// <summary> /// Initializes a new instance of the <c>JsonLexer</c> class. /// </summary> /// <param name="classificationTypeProvider">A <see cref="IJsonClassificationTypeProvider"/> that provides classification types used by this lexer.</param> public JsonLexer(IJsonClassificationTypeProvider classificationTypeProvider) { if ((classificationTypeProvider == null)) { throw new ArgumentNullException("classificationTypeProvider"); } // Create lexical states IKeyedObservableCollection<DynamicLexicalState> lexicalStates = ((IKeyedObservableCollection<DynamicLexicalState>)(this.LexicalStates)); this.DefaultLexicalState.Id = JsonLexicalStateId.Default; lexicalStates.Add(new DynamicLexicalState(JsonLexicalStateId.PrimaryString, "PrimaryString")); DynamicLexicalState lexicalState = null; DynamicLexicalScope lexicalScope = null; DynamicLexicalPatternGroup lexicalPatternGroup = null; // Initialize the Default lexical state lexicalState = lexicalStates["Default"]; lexicalState.ChildLexicalStates.Add(lexicalStates["PrimaryString"]); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Whitespace", null); lexicalPatternGroup.TokenId = JsonTokenId.Whitespace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{LineTerminatorWhitespace}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Comma", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.Comma; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(",")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Colon", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.Colon; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(":")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.OpenCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.CloseCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenSquareBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.OpenSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseSquareBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = JsonTokenId.CloseSquareBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("]")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "True", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.True; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("true")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "False", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.False; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("false")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Null", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = JsonTokenId.Null; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.AutoCorrect; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("null")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Number", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = JsonTokenId.Number; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}* \\. {Digit}+ ([Ee] [\\+\\-]? {Digit}+)?")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}+ [Ee] [\\+\\-]? {Digit}+")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[\\+\\-]?{Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the PrimaryString lexical state lexicalState = lexicalStates["PrimaryString"]; lexicalState.DefaultClassificationType = classificationTypeProvider.String; lexicalState.DefaultTokenId = JsonTokenId.PrimaryStringText; lexicalState.DefaultTokenKey = "PrimaryStringText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "StringStartDelimiter", null); lexicalPatternGroup.TokenId = JsonTokenId.StringStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "StringEndDelimiter", null); lexicalPatternGroup.TokenId = JsonTokenId.StringEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\"")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "EscapedCharacter", null); lexicalPatternGroup.TokenId = JsonTokenId.EscapedCharacter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\"")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\\")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\/")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\b")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\f")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\n")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\r")); lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\t")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "EscapedUnicode", null); lexicalPatternGroup.TokenId = JsonTokenId.EscapedUnicode; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\\\u[0-9a-fA-F]{4}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "StringText", null); lexicalPatternGroup.TokenId = JsonTokenId.StringText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[^\\\"\\\\\\n]+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); }
/// <summary> /// Initializes a new instance of the <c>SimpleLexer</c> class. /// </summary> /// <param name="classificationTypeProvider">A <see cref="ISimpleClassificationTypeProvider"/> that provides classification types used by this lexer.</param> public SimpleLexer(ISimpleClassificationTypeProvider classificationTypeProvider) { if ((classificationTypeProvider == null)) { throw new ArgumentNullException("classificationTypeProvider"); } // Create lexical states IKeyedObservableCollection <DynamicLexicalState> lexicalStates = ((IKeyedObservableCollection <DynamicLexicalState>)(this.LexicalStates)); this.DefaultLexicalState.Id = SimpleLexicalStateId.Default; lexicalStates.Add(new DynamicLexicalState(SimpleLexicalStateId.SingleLineComment, "SingleLineComment")); lexicalStates.Add(new DynamicLexicalState(SimpleLexicalStateId.MultiLineComment, "MultiLineComment")); DynamicLexicalState lexicalState = null; DynamicLexicalScope lexicalScope = null; DynamicLexicalPatternGroup lexicalPatternGroup = null; // Initialize the Default lexical state lexicalState = lexicalStates["Default"]; lexicalState.ChildLexicalStates.Add(lexicalStates["SingleLineComment"]); lexicalState.ChildLexicalStates.Add(lexicalStates["MultiLineComment"]); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Whitespace", null); lexicalPatternGroup.TokenId = SimpleTokenId.Whitespace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{LineTerminatorWhitespace}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenParenthesis", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = SimpleTokenId.OpenParenthesis; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("(")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseParenthesis", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = SimpleTokenId.CloseParenthesis; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(")")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "OpenCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = SimpleTokenId.OpenCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "CloseCurlyBrace", classificationTypeProvider.Delimiter); lexicalPatternGroup.TokenId = SimpleTokenId.CloseCurlyBrace; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("}")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Function", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = SimpleTokenId.Function; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.Sensitive; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("function")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Return", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = SimpleTokenId.Return; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.Sensitive; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("return")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Var", classificationTypeProvider.Keyword); lexicalPatternGroup.TokenId = SimpleTokenId.Var; lexicalPatternGroup.CaseSensitivity = CaseSensitivity.Sensitive; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("var")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Identifier", classificationTypeProvider.Identifier); lexicalPatternGroup.TokenId = SimpleTokenId.Identifier; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("(_ | {Alpha})({Word})*")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Equality", null); lexicalPatternGroup.TokenId = SimpleTokenId.Equality; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("==")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Inequality", null); lexicalPatternGroup.TokenId = SimpleTokenId.Inequality; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("!=")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Assignment", null); lexicalPatternGroup.TokenId = SimpleTokenId.Assignment; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("=")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Addition", null); lexicalPatternGroup.TokenId = SimpleTokenId.Addition; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Subtraction", null); lexicalPatternGroup.TokenId = SimpleTokenId.Subtraction; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("-")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Multiplication", null); lexicalPatternGroup.TokenId = SimpleTokenId.Multiplication; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("*")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Division", null); lexicalPatternGroup.TokenId = SimpleTokenId.Division; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("/")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "Comma", null); lexicalPatternGroup.TokenId = SimpleTokenId.Comma; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(",")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "SemiColon", null); lexicalPatternGroup.TokenId = SimpleTokenId.SemiColon; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern(";")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Number", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = SimpleTokenId.Number; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{Digit}* \\. {Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "Number", classificationTypeProvider.Number); lexicalPatternGroup.TokenId = SimpleTokenId.Number; lexicalPatternGroup.LookAheadPattern = "{NonWord}|\\z"; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("{Digit}+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the SingleLineComment lexical state lexicalState = lexicalStates["SingleLineComment"]; lexicalState.DefaultClassificationType = classificationTypeProvider.Comment; lexicalState.DefaultTokenId = SimpleTokenId.SingleLineCommentText; lexicalState.DefaultTokenKey = "SingleLineCommentText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "SingleLineCommentStartDelimiter", null); lexicalPatternGroup.TokenId = SimpleTokenId.SingleLineCommentStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("//")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "SingleLineCommentEndDelimiter", null); lexicalPatternGroup.TokenId = SimpleTokenId.SingleLineCommentEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\n")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "SingleLineCommentText", null); lexicalPatternGroup.TokenId = SimpleTokenId.SingleLineCommentText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[^\\n]+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); // Initialize the MultiLineComment lexical state lexicalState = lexicalStates["MultiLineComment"]; lexicalState.DefaultClassificationType = classificationTypeProvider.Comment; lexicalState.DefaultTokenId = SimpleTokenId.MultiLineCommentText; lexicalState.DefaultTokenKey = "MultiLineCommentText"; lexicalScope = new DynamicLexicalScope(); lexicalState.LexicalScopes.Add(lexicalScope); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentStartDelimiter", null); lexicalPatternGroup.TokenId = SimpleTokenId.MultiLineCommentStartDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("/*")); lexicalScope.StartLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Explicit, "MultiLineCommentEndDelimiter", null); lexicalPatternGroup.TokenId = SimpleTokenId.MultiLineCommentEndDelimiter; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("*/")); lexicalScope.EndLexicalPatternGroup = lexicalPatternGroup; lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "MultiLineCommentLineTerminator", null); lexicalPatternGroup.TokenId = SimpleTokenId.MultiLineCommentLineTerminator; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("\\n")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); lexicalPatternGroup = new DynamicLexicalPatternGroup(DynamicLexicalPatternType.Regex, "MultiLineCommentText", null); lexicalPatternGroup.TokenId = SimpleTokenId.MultiLineCommentText; lexicalPatternGroup.Patterns.Add(new DynamicLexicalPattern("[^\\*\\n]+")); lexicalState.LexicalPatternGroups.Add(lexicalPatternGroup); }