/// <summary> /// Verify a source code and return the Inode /// </summary> /// <param name="source"></param> /// <returns></returns> public INode Verifiy(string source) { RulezParser.MessageListener aListener = new RulezParser.MessageListener(); RulezParser.RulezUnitContext aCtx = null; try { RulezLexer aLexer = new RulezLexer(new Antlr4.Runtime.AntlrInputStream(source)); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this; aParser.AddErrorListener(aListener); aCtx = aParser.rulezUnit(); if (aCtx != null) { return(aCtx.XPTreeNode); } return(null); } catch (Exception ex) { if (aCtx != null) { if (aCtx.XPTreeNode != null) { aCtx.XPTreeNode.Messages.Add(new Message(type: MessageType.Error, message: ex.Message)); return(aCtx.XPTreeNode); } } return(null); } }
private TemplateParser.GroupFileContext ParseWithAntlr4(ITextSnapshot snapshot, out IList <Antlr4.Runtime.IToken> tokensList) { var input = new Parsing4.SnapshotCharStream(snapshot, new Span(0, snapshot.Length)); var lexer = new TemplateLexer(input); var tokens = new Antlr4.Runtime.CommonTokenStream(lexer); var parser = new TemplateParser(tokens); TemplateParser.GroupFileContext parseResult; try { parser.Interpreter.PredictionMode = Antlr4.Runtime.Atn.PredictionMode.Sll; parser.RemoveErrorListeners(); parser.BuildParseTree = true; parser.ErrorHandler = new Antlr4.Runtime.BailErrorStrategy(); parseResult = parser.groupFile(); } catch (Antlr4.Runtime.Misc.ParseCanceledException ex) when(ex.InnerException is Antlr4.Runtime.RecognitionException) { // retry with default error handler tokens.Reset(); parser.Interpreter.PredictionMode = Antlr4.Runtime.Atn.PredictionMode.Ll; //parser.AddErrorListener(DescriptiveErrorListener.INSTANCE); parser.SetInputStream(tokens); parser.ErrorHandler = new Antlr4.Runtime.DefaultErrorStrategy(); parseResult = parser.groupFile(); } tokensList = tokens.GetTokens(); return(parseResult); }
} // End Sub WalkerTest static void VisitorTest(string text) { try { System.IO.StringReader reader = new System.IO.StringReader(text); Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokens = new Antlr4.Runtime.CommonTokenStream(lexer); TSqlParser parser = new TSqlParser(tokens); //Specify our entry point // TSqlParser.Query_specificationContext TSqlParser.Tsql_fileContext fileContext = parser.tsql_file(); System.Console.WriteLine("fileContext.ChildCount = " + fileContext.ChildCount.ToString()); SqlVisitor vis = new SqlVisitor(); string s = vis.Visit(fileContext); System.Console.WriteLine(s); } catch (System.Exception e) { System.Console.WriteLine(e.Message); } } // End Sub VisitorTest
} // End Sub LexerTest static void WalkerTest(string text) { try { System.IO.StringReader reader = new System.IO.StringReader(text); Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokens = new Antlr4.Runtime.CommonTokenStream(lexer); TSqlParser parser = new TSqlParser(tokens); // Specify our entry point // TSqlParser.Query_specificationContext TSqlParser.Tsql_fileContext fileContext = parser.tsql_file(); // Antlr4.Runtime.Tree.IParseTree root = (Antlr4.Runtime.Tree.IParseTree)fileContext; // TSqlParser.Query_specificationContext tsqlParser.Tsql_fileContext fileContext = parser.tsql_file(); System.Console.WriteLine("fileContext.ChildCount = " + fileContext.ChildCount.ToString()); // Walk it and attach our listener Antlr4.Runtime.Tree.ParseTreeWalker walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); // AntlrTsqListener listener = new AntlrTsqListener(); EverythingListener listener = new EverythingListener(); walker.Walk(listener, fileContext); } catch (System.Exception e) { System.Console.WriteLine(e.Message); } } // End Sub WalkerTest
public void ParseInputFile(System.IO.StreamReader fIn) { var input = new Antlr4.Runtime.AntlrInputStream(fIn); var lexer = new Grammars.ZeroCode2Lexer(input); var tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); var parser = new Grammars.ZeroCode2(tokenStream); var walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); var Listener = new ZeroCodeListener(); walker.Walk(Listener, parser.zcDefinition()); ModelCollector = Listener.Collector; GraphElements = Listener.GraphElements; if (ResolveInheritance() == false) { logger.Error("Not all Inheritances were resolved"); HasErrors = true; } if (parser.NumberOfSyntaxErrors > 0) { HasErrors = true; logger.Error("Errors exist in the input file"); } // dump all errors: Errors.ForEach(e => logger.Error(e)); }
/// <summary> /// generate from a input string a rule by parsing and compiling and store it /// </summary> /// <param name="input"></param> /// <returns></returns> private bool Generate(Antlr4.Runtime.ICharStream input) { try { RulezLexer aLexer = new RulezLexer(input); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this; //aParser.RemoveErrorListeners(); //aParser.AddErrorListener(new ErrorListener()); RulezParser.RulezUnitContext aTree = aParser.rulezUnit(); // walk the parse tree -> get the result XPTree XPTGenerator aGenerator = new XPTGenerator(aParser); Antlr4.Runtime.Tree.ParseTreeWalker.Default.Walk(aGenerator, aTree); // result -> Generate and store from the XPTree return(Generate((IRule)aGenerator.XPTree)); } catch (Exception ex) { return(false); } }
static void Main(string[] args) { var inputStream = new Antlr4.Runtime.AntlrFileStream(args[0]); var lexer = new Antlr4.Languages.Java.Java8Lexer(inputStream); var ts = new Antlr4.Runtime.CommonTokenStream(lexer); var parser = new Antlr4.Languages.Java.Java8Parser(ts); var walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); }
public static System.Collections.Generic.List <string> GetVariableNames() { System.Collections.Generic.List <string> ls = new System.Collections.Generic.List <string>(); string text = @" SELECT BE_Name FROM T_Benutzer WHERE Name =@username OR Name LIKE '%' + @foo + '%' "; System.IO.StringReader reader = new System.IO.StringReader(text); // Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.ICharStream input1 = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.CaseChangingCharStream input = new Antlr4.Runtime.CaseChangingCharStream(input1, true); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); tokenStream.Fill(); int lastIndex = 0; foreach (Antlr4.Runtime.IToken token in tokenStream.GetTokens()) { // System.Console.WriteLine(token.Text); string tokenTypeName = lexer.Vocabulary.GetSymbolicName(token.Type); Antlr4.Runtime.Misc.Interval ival = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extracted = token.InputStream.GetText(ival); // table_name, cte_name: ID, SQUARE_BRACKET_ID // Local variables: LOCAL_ID if (token.Type == TSqlLexer.LOCAL_ID) { extracted = extracted.Trim(new char[] { ' ', '\t', '\v', '\r', '\n' }); ls.Add(extracted); } // End if (token.Type == TSqlLexer.LOCAL_ID) lastIndex = token.StopIndex + 1; } // Next token return(ls); }
public IEnumerable <LintInfo> Lint(Stream stream, ProjectFile file) { var inputStream = new Antlr4.Runtime.AntlrInputStream(stream); var lexer = new RealVirtuality.Config.Parser.ANTLR.armaconfigLexer(inputStream); var commonTokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); var p = new RealVirtuality.Config.Parser.ANTLR.armaconfigParser(commonTokenStream); var listener = new RealVirtuality.Config.Parser.ANTLR.ArmaConfigListener(); p.AddParseListener(listener); p.RemoveErrorListeners(); #if DEBUG p.BuildParseTree = true; p.Context = new Antlr4.Runtime.ParserRuleContext(); #endif var se = new List <LintInfo>(); p.AddErrorListener(new RealVirtuality.Config.Parser.ANTLR.ErrorListener((recognizer, token, line, charPositionInLine, msg, ex) => { se.Add(new LintInfo(file) { StartOffset = token.StartIndex, Length = token.Text.Length, Message = msg, Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine }); })); try { p.armaconfig(); } catch (Exception ex) { App.ShowOperationFailedMessageBox(ex); while (ex.InnerException != null) { ex = ex.InnerException; App.ShowOperationFailedMessageBox(ex); } System.Diagnostics.Debugger.Break(); } return(se); }
public void ParseTemplateFile(System.IO.StreamReader fIn) { var input = new Antlr4.Runtime.AntlrInputStream(fIn); var lexer = new Grammars.ZeroCode2TemplateLexer(input); var tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); var parser = new Grammars.ZeroCode2Template(tokenStream); var walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); var Listener = new ZeroCode2TemplateListener { Program = Program }; walker.Walk(Listener, parser.template()); Errors.AddRange(Program.Errors()); HasErrors = Errors.Count > 0; }
public override void ExitDatatableRowCell([NotNull] SBP.DatatableRowCellContext context) { var text = context.GetText().Trim(); TypeReference type = null; object value = null; if (!String.IsNullOrEmpty(text)) { // TODO: It MUST be possible to make this simpler... (e.g. re-use tokens) Antlr4.Runtime.ITokenSource lexer = new Lexer(new Antlr4.Runtime.AntlrInputStream(text)); Antlr4.Runtime.ITokenStream tokens = new Antlr4.Runtime.CommonTokenStream(lexer); var parser = new SBP(tokens); parser.RemoveErrorListeners(); parser.AddErrorListener(m_errors); parser.BuildParseTree = true; var cellContext = parser.datatableRowCellContent(); var walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); m_expressionData.PushStackLevel("VariableType"); walker.Walk(this, cellContext); var stack = m_expressionData.PopStackLevel(); var expressionValue = stack.Pop(); if (expressionValue.IsConstant) { type = expressionValue.DataType; value = expressionValue.Value; } else if (expressionValue.IsUnresolvedIdentifier) { type = new TypeReference(typeof(StepBro.Core.Data.Identifier)); value = expressionValue.Value; } else { m_errors.SymanticError(context.Start.Line, context.Start.Column, false, "Error parsing cell value."); } } m_currentDatatableRowData.Add(new Tuple <string, TypeReference, object>(text, type, value)); }
//выполнить запрос private void requestBtn_Click(object sender, EventArgs e) { resultBox.Text = ""; resultBox.Text = ""; requestList.AddStr(requestBox.Text); Antlr4.Runtime.ICharStream input = new Antlr4.Runtime.AntlrInputStream(requestBox.Text); PreSQLLexer lexer = new PreSQLLexer(input); Antlr4.Runtime.CommonTokenStream tokens = new Antlr4.Runtime.CommonTokenStream(lexer); PreSQLParser parser = new PreSQLParser(tokens); parser.RemoveErrorListeners(); // remove ConsoleErrorListener parser.AddErrorListener(new ErrorListener()); // добавит листенер, который будет сообщать об ошибках распарсивания //parser.AddParseListener(new Listener()); // add ours parser.BuildParseTree = true; Antlr4.Runtime.Tree.IParseTree tree = parser.parse(); // получить дерево resultBox.Text += tree.ToStringTree(parser) + Environment.NewLine + Environment.NewLine; // print LISP-style tree // Анализируем результаты разбора. Если были синтаксические ошибки, выдадим лог анализа if (parser.NumberOfSyntaxErrors != 0) { resultBox.Text += "> Parsing Error!" + Environment.NewLine; } else {//строим дерево treeView.Nodes.Clear(); th.LoadNode(null, tree); resultBox.Text += "> Parsing Ok!" + Environment.NewLine; Visitor visitor = new Visitor(this); //запускаем Визитор // он проходит по дереву, определяет SQL-оператор и выполняет действие в зависимости(см. файлVisitor) visitor.Visit(tree); } }
} // End Sub CommentRemoverLexerTest // https://github.com/dotjpg3141/Strings static void LexerTest(string text) { try { System.IO.StringReader reader = new System.IO.StringReader(text); // Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.ICharStream input1 = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.CaseChangingCharStream input = new Antlr4.Runtime.CaseChangingCharStream(input1, true); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); tokenStream.Fill(); System.Text.StringBuilder sb = new System.Text.StringBuilder(); int lastIndex = 0; foreach (Antlr4.Runtime.IToken token in tokenStream.GetTokens()) { // System.Console.WriteLine(token.Text); string tokenTypeName = lexer.Vocabulary.GetSymbolicName(token.Type); if (token.Type == TSqlLexer.LINE_COMMENT || token.Type == TSqlLexer.COMMENT || token.Type == TSqlLexer.Eof) { Antlr4.Runtime.Misc.Interval blankInterval = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StartIndex - 1); string extractedBlank = token.InputStream.GetText(blankInterval); if (string.IsNullOrEmpty(extractedBlank)) { sb.Append(" "); } else { sb.Append(extractedBlank); } lastIndex = token.StopIndex + 1; continue; } // End if comment // sql += token.Text + " "; Antlr4.Runtime.Misc.Interval ival = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extracted = token.InputStream.GetText(ival); // System.Console.WriteLine((extracted)); sb.Append(extracted); // System.Console.WriteLine(token.Text); // System.Console.WriteLine(token.Type); // System.Console.WriteLine(tokenTypeName); lastIndex = token.StopIndex + 1; } // Next token string sql = sb.ToString(); sb.Clear(); sb = null; System.Console.WriteLine(sql); } catch (System.Exception e) { System.Console.WriteLine(e.Message); } } // End Sub LexerTest
/// <summary> /// Verify a source code and return the Inode /// </summary> /// <param name="source"></param> /// <returns></returns> public INode Verifiy(string source) { RulezParser.MessageListener aListener = new RulezParser.MessageListener(); RulezParser.RulezUnitContext aCtx = null; try { RulezLexer aLexer = new RulezLexer(new Antlr4.Runtime.AntlrInputStream(source)); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this; aParser.AddErrorListener(aListener); aCtx = aParser.rulezUnit(); if (aCtx != null) return aCtx.XPTreeNode; return null; } catch (Exception ex) { if (aCtx != null) { if (aCtx.XPTreeNode != null) { aCtx.XPTreeNode.Messages.Add(new Message(type: MessageType.Error, message: ex.Message)); return aCtx.XPTreeNode; } } return null; } }
/// <summary> /// generate a selection Rule XPT out of a String /// </summary> /// <param name="source"></param> /// <returns></returns> public SelectionRule Generate(string source) { RulezParser.MessageListener aListener = new RulezParser.MessageListener(); RulezParser.SelectionRulezContext aCtx = null; try { RulezLexer aLexer = new RulezLexer(new Antlr4.Runtime.AntlrInputStream(source)); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this.Engine; aParser.AddErrorListener(aListener); // parse aCtx = aParser.selectionRulez(); // return return (SelectionRule) aCtx.XPTreeNode; } catch (Exception ex) { if (aCtx != null) return (SelectionRule) aCtx.XPTreeNode; return null; } }
public IEnumerable <LintInfo> Lint(Stream stream, ProjectFile file) { var inputStream = new Antlr4.Runtime.AntlrInputStream(stream); var lexer = new RealVirtuality.SQF.ANTLR.Parser.sqfLexer(inputStream); var commonTokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); var p = new RealVirtuality.SQF.ANTLR.Parser.sqfParser(commonTokenStream, ConfigHost.Instance.SqfDefinitions); var listener = new RealVirtuality.SQF.ANTLR.SqfListener(); p.AddParseListener(listener); p.RemoveErrorListeners(); #if DEBUG p.BuildParseTree = true; p.Context = new Antlr4.Runtime.ParserRuleContext(); #endif var se = new List <LintInfo>(); p.AddErrorListener(new RealVirtuality.SQF.ANTLR.ErrorListener((recognizer, token, line, charPositionInLine, msg, ex) => { switch (ex == null ? null : p.RuleNames[ex.Context.RuleIndex]) { case "binaryexpression": se.Add(new LintInfo(file) { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid binary expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine }); break; case "unaryexpression": se.Add(new LintInfo(file) { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid unary expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine }); break; case "nularexpression": se.Add(new LintInfo(file) { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid nular expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine }); break; default: se.Add(new LintInfo(file) { StartOffset = token.StartIndex, Length = token.Text.Length, Message = msg, Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine }); break; } })); try { p.sqf(); } catch (Exception ex) { App.ShowOperationFailedMessageBox(ex); while (ex.InnerException != null) { ex = ex.InnerException; App.ShowOperationFailedMessageBox(ex); } System.Diagnostics.Debugger.Break(); } return(se); }
} // End Sub SplitMultiTableInsertScript static string SubstituteVariablesTest() { // https://github.com/antlr/grammars-v4/tree/master/tsql // https://github.com/antlr/grammars-v4/tree/master/plsql/CSharp string text = @" DECLARE @legalEntity int -- SET @legalEntity = 1 ;WITH CTE AS ( SELECT 1 AS id, 123 AS abc UNION SELECT 2 AS id, 456 AS abc UNION SELECT 3 AS id, 789 AS abc UNION SELECT 4 AS id, 012 AS abc UNION SELECT 5 AS id, 345 AS abc UNION SELECT 6 AS id, 678 AS abc ) SELECT * ,'@legalEntity' AS abcdef -- strings do not get substituted ,987 AS [@legalEntity] -- identifiers do not get substituted FROM CTE WHERE (1=1) AND ( '0' IN (@legalEntity, @legalEntity ) OR CTE.id IN (@legalEntity) -- CTE.id IN (@legalEntity /* @legalEntity */) ) /* ==> AND ( '0' IN (1,2,6) OR CTE.id IN (1,2,6) -- OR CTE.id IN (1,2,3,4,5,6 /* 1,2,3,4,5,6 */) ) */ "; System.IO.StringReader reader = new System.IO.StringReader(text); // Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.ICharStream input1 = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.CaseChangingCharStream input = new Antlr4.Runtime.CaseChangingCharStream(input1, true); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); tokenStream.Fill(); System.Text.StringBuilder sb = new System.Text.StringBuilder(); int lastIndex = 0; foreach (Antlr4.Runtime.IToken token in tokenStream.GetTokens()) { // System.Console.WriteLine(token.Text); string tokenTypeName = lexer.Vocabulary.GetSymbolicName(token.Type); #if NO_COMMENTS if (token.Type == TSqlLexer.LINE_COMMENT || token.Type == TSqlLexer.COMMENT || token.Type == TSqlLexer.Eof) { Antlr4.Runtime.Misc.Interval blankInterval = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StartIndex - 1); string extractedBlank = token.InputStream.GetText(blankInterval); if (string.IsNullOrEmpty(extractedBlank)) { sb.Append(" "); } else { sb.Append(extractedBlank); } lastIndex = token.StopIndex + 1; continue; } // End if comment #endif // sql += token.Text + " "; Antlr4.Runtime.Misc.Interval ival = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extracted = token.InputStream.GetText(ival); // table_name, cte_name: ID, SQUARE_BRACKET_ID // Local variables: LOCAL_ID if (token.Type == TSqlLexer.LOCAL_ID) { extracted = extracted.Trim(new char[] { ' ', '\t', '\v', '\r', '\n' }); System.Console.WriteLine(extracted); } // End if (token.Type == TSqlLexer.LOCAL_ID) // System.Console.WriteLine((extracted)); sb.Append(extracted); // System.Console.WriteLine(token.Text); // System.Console.WriteLine(token.Type); // System.Console.WriteLine(tokenTypeName); lastIndex = token.StopIndex + 1; } // Next token string sql = sb.ToString(); sb.Clear(); sb = null; System.Console.WriteLine(sql); return(sql); } // End Sub SubstituteVariablesTest
protected override IEnumerable <LinterInfo> GetLinterInformations(MemoryStream memstream) { var inputStream = new Antlr4.Runtime.AntlrInputStream(memstream); var lexer = new RealVirtuality.SQF.ANTLR.Parser.sqfLexer(inputStream); var commonTokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); var p = new RealVirtuality.SQF.ANTLR.Parser.sqfParser(commonTokenStream, ConfigHost.Instance.SqfDefinitions); var listener = new RealVirtuality.SQF.ANTLR.SqfListener(); p.AddParseListener(listener); memstream.Seek(0, SeekOrigin.Begin); p.RemoveErrorListeners(); #if DEBUG p.BuildParseTree = true; p.Context = new Antlr4.Runtime.ParserRuleContext(); #endif var se = new List <LinterInfo>(); p.AddErrorListener(new RealVirtuality.SQF.ANTLR.ErrorListener((recognizer, token, line, charPositionInLine, msg, ex) => { switch (ex == null ? null : p.RuleNames[ex.Context.RuleIndex]) { case "binaryexpression": se.Add(new LinterInfo() { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid binary expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine, FilePath = this.FilePath }); break; case "unaryexpression": se.Add(new LinterInfo() { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid unary expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine, FilePath = this.FilePath }); break; case "nularexpression": se.Add(new LinterInfo() { StartOffset = token.StartIndex, Length = token.Text.Length, Message = string.Concat("Invalid nular expression: ", msg), Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine, FilePath = this.FilePath }); break; default: se.Add(new LinterInfo() { StartOffset = token.StartIndex, Length = token.Text.Length, Message = msg, Severity = ESeverity.Error, Line = line, LineOffset = charPositionInLine, FilePath = this.FilePath }); break; } })); p.sqf(); return(se); }
/// <summary> /// generate from a input string a rule by parsing and compiling and store it /// </summary> /// <param name="input"></param> /// <returns></returns> private bool Generate(Antlr4.Runtime.ICharStream input) { try { RulezLexer aLexer = new RulezLexer(input); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this; //aParser.RemoveErrorListeners(); //aParser.AddErrorListener(new ErrorListener()); RulezParser.RulezUnitContext aTree = aParser.rulezUnit(); // walk the parse tree -> get the result XPTree XPTGenerator aGenerator = new XPTGenerator(aParser); Antlr4.Runtime.Tree.ParseTreeWalker.Default.Walk(aGenerator, aTree); // result -> Generate and store from the XPTree return Generate((IRule) aGenerator.XPTree); } catch (Exception ex) { return false; } }
public static System.Collections.Generic.List <string> LexWithAntlr(string text) { System.Collections.Generic.List <string> ls = new System.Collections.Generic.List <string>(); System.IO.StringReader reader = new System.IO.StringReader(text); // Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.ICharStream input1 = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.CaseChangingCharStream input = new Antlr4.Runtime.CaseChangingCharStream(input1, true); Rfc5424Lexer lexer = new Rfc5424Lexer(input); Antlr4.Runtime.CommonTokenStream tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); tokenStream.Fill(); Rfc5424Parser parser = new Rfc5424Parser(tokenStream); Rfc5424Parser.Syslog_msgContext msgContext = parser.syslog_msg(); RfcVisitor vis = new RfcVisitor(); string s = vis.Visit(msgContext); Antlr4.Runtime.Tree.ParseTreeWalker walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); // AntlrTsqListener listener = new AntlrTsqListener(); EverythingListener listener = new EverythingListener(); // walker.Walk(listener, msgContext); // new EverythingListener().EnterBom(parser.bom()); // new EverythingListener().EnterTimestamp(parser.timestamp()); // new EverythingListener().EnterEveryRule(parser.version()); // new EverythingListener().EnterEveryRule(parser.timestamp()); // var x = parser.msg(); var x = parser.timestamp(); Antlr4.Runtime.Misc.Interval msgInt = x.SourceInterval; // new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extractedMsg = tokenStream.GetText(msgInt); System.Console.WriteLine(extractedMsg); int lastIndex = 0; foreach (Antlr4.Runtime.IToken token in tokenStream.GetTokens()) { // System.Console.WriteLine(token.Text); string tokenTypeName = lexer.Vocabulary.GetSymbolicName(token.Type); Antlr4.Runtime.Misc.Interval ival = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extracted = token.InputStream.GetText(ival); // table_name, cte_name: ID, SQUARE_BRACKET_ID // Local variables: LOCAL_ID lastIndex = token.StopIndex + 1; } // Next token return(ls); }
static void SplitMultiTableInsertScript() { string fileName = @"D:\SQL\TESS\Anlage_Refdaten.txt"; fileName = @"D:\SQL\TESS\Adressdaten.txt"; fileName = @"D:\SQL\TESS\Anlagedaten.txt"; fileName = @"D:\SQL\TESS\Anlagerechte.txt"; fileName = @"D:\SQL\TESS\Kontaktdaten.txt"; fileName = @"D:\SQL\TESS\Navigation.txt"; fileName = @"D:\username\Desktop\Raumdaten\Raumdaten.sql"; fileName = @"D:\username\Desktop\Raumdaten\Vertragsdaten.sql"; System.Text.StringBuilder sb = new System.Text.StringBuilder(); // https://github.com/antlr/grammars-v4/tree/master/tsql // https://github.com/antlr/grammars-v4/tree/master/plsql/CSharp System.Text.Encoding enc = GetSystemEncoding(); string text = System.IO.File.ReadAllText(fileName, enc); System.IO.StringReader reader = new System.IO.StringReader(text); // Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.ICharStream input1 = new Antlr4.Runtime.AntlrInputStream(reader); Antlr4.Runtime.CaseChangingCharStream input = new Antlr4.Runtime.CaseChangingCharStream(input1, true); TSqlLexer lexer = new TSqlLexer(input); Antlr4.Runtime.CommonTokenStream tokenStream = new Antlr4.Runtime.CommonTokenStream(lexer); tokenStream.Fill(); int lastIndex = 0; System.Collections.Generic.Dictionary <string, System.Collections.Generic.List <string> > dict = new System.Collections.Generic.Dictionary <string, System.Collections.Generic.List <string> >(System.StringComparer.InvariantCultureIgnoreCase); System.Collections.Generic.List <string> lsTableName = new System.Collections.Generic.List <string>(); bool ignoreThis = true; bool partOfTableName = false; int lastTokenType = -1; int secondLastTokenType = -1; foreach (Antlr4.Runtime.IToken token in tokenStream.GetTokens()) { // System.Console.WriteLine(token.Text); string tokenTypeName = lexer.Vocabulary.GetSymbolicName(token.Type); Antlr4.Runtime.Misc.Interval ival = new Antlr4.Runtime.Misc.Interval(lastIndex, token.StopIndex); string extracted = token.InputStream.GetText(ival); extracted = extracted.Trim(new char[] { '\t', '\v', '\r', '\n' }); if (token.Type == TSqlLexer.INSERT) { if (sb.Length > 0) { string tn = string.Join(".", lsTableName.ToArray()).Replace("[", "").Replace("]", "").Trim(); lsTableName.Clear(); System.Console.WriteLine(tn); if (!dict.ContainsKey(tn)) { dict[tn] = new System.Collections.Generic.List <string>(); } sb.Append(";"); dict[tn].Add(sb.ToString()); } sb.Clear(); ignoreThis = false; partOfTableName = true; } else if (token.Type == TSqlLexer.GO) { ignoreThis = true; partOfTableName = false; } else if (token.Type == TSqlLexer.USE) { ignoreThis = true; partOfTableName = false; } else if (token.Type == TSqlLexer.SEMI) { ignoreThis = true; partOfTableName = false; } else if (token.Type == TSqlLexer.Eof) { } else if (token.Type == TSqlLexer.LR_BRACKET) { partOfTableName = false; } else if (token.Type == TSqlLexer.RR_BRACKET) { } else if (token.Type == TSqlLexer.COMMA) { } else if (token.Type == TSqlLexer.INTO) { } else if (token.Type == TSqlLexer.VALUES || token.Type == TSqlLexer.SELECT) { } else if (token.Type == TSqlLexer.ID || token.Type == TSqlLexer.SQUARE_BRACKET_ID) { if (partOfTableName) { lsTableName.Add(extracted); } } else if (token.Type == TSqlLexer.DOT) { } else if (token.Type == TSqlLexer.STRING) { } else if (token.Type == TSqlLexer.DECIMAL) { } else if (token.Type == TSqlLexer.FLOAT) { } else if (token.Type == TSqlLexer.NULL) { } else if (token.Type == TSqlLexer.CAST) { } else if (token.Type == TSqlLexer.AS) { // CAST(xxx AS datetime) } else if (token.Type == TSqlLexer.MINUS) { // Negative Number } else { System.Console.WriteLine(tokenTypeName); } // System.Console.WriteLine((extracted)); if (!ignoreThis && token.Type != TSqlLexer.SEMI) { sb.Append(extracted); } // System.Console.WriteLine(token.Text); // System.Console.WriteLine(token.Type); // System.Console.WriteLine(tokenTypeName); lastIndex = token.StopIndex + 1; secondLastTokenType = lastTokenType; lastTokenType = token.Type; } // Next token if (sb.Length > 0) { string tn = string.Join(".", lsTableName.ToArray()).Replace("[", "").Replace("]", "").Trim(); lsTableName.Clear(); System.Console.WriteLine(tn); if (!dict.ContainsKey(tn)) { dict[tn] = new System.Collections.Generic.List <string>(); } sb.Append(";"); dict[tn].Add(sb.ToString()); } // End if (sb.Length > 0) sb.Clear(); sb = null; string baseDir = System.IO.Path.GetFileNameWithoutExtension(fileName); string outputDirectory = System.IO.Path.GetDirectoryName(fileName); baseDir = System.IO.Path.Combine(outputDirectory, baseDir); if (!System.IO.Directory.Exists(baseDir)) { System.IO.Directory.CreateDirectory(baseDir); } foreach (System.Collections.Generic.KeyValuePair <string, System.Collections.Generic.List <string> > kvp in dict) { string dir = kvp.Key; string content = string.Join("\r\n\r\n", kvp.Value.ToArray()); System.Console.WriteLine(content); string fn = System.IO.Path.Combine(baseDir, kvp.Key + ".sql"); System.IO.File.WriteAllText(fn, content, System.Text.Encoding.UTF8); } // Next kvp System.Console.WriteLine(dict); } // End Sub SplitMultiTableInsertScript