public static MySQL51Parser.program_return ParseSql(string sql, bool expectErrors, out StringBuilder sb) { // The grammar supports upper case only MemoryStream ms = new MemoryStream(ASCIIEncoding.ASCII.GetBytes(sql/*.ToUpper() */)); CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms); //ANTLRInputStream input = new ANTLRInputStream(ms); MySQL51Lexer lexer = new MySQL51Lexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); MySQL51Parser parser = new MySQL51Parser(tokens); sb = new StringBuilder(); TextWriter tw = new StringWriter(sb); parser.TraceDestination = tw; MySQL51Parser.program_return r = parser.program(); if (!expectErrors) { if (0 != parser.NumberOfSyntaxErrors) Assert.AreEqual("", sb.ToString()); //Assert.AreEqual( 0, parser.NumberOfSyntaxErrors); } else { Assert.AreNotEqual(0, parser.NumberOfSyntaxErrors); } return r; }
public static TokenStreamRemovable GetTokenStream(string sql, Version version) { Dictionary <Version, TokenStreamRemovable> lines; if (_parserCache.TryGetValue(sql, out lines)) { TokenStreamRemovable tsr; if (lines.TryGetValue(version, out tsr)) { return(tsr); } } // no cache entry, then parse MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(sql));//ASCIIEncoding.ASCII.GetBytes(sql)); CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms); MySQLLexer lexer = new MySQLLexer(input); lexer.MySqlVersion = version; TokenStreamRemovable tokens = new TokenStreamRemovable(lexer); if (lines == null) { lines = new Dictionary <Version, TokenStreamRemovable>(); _parserCache.Add(sql, lines); } lines.Add(version, tokens); return(tokens); }
public void CaseInsensitiveLexerTest() { var stream = new CaseInsensitiveInputStream(data); var lexer = new CaseInsensitiveLexer(stream); lexer.AddErrorListener(errorListener); var tokens = lexer.GetAllTokens(); }
internal static MySQL51Parser.program_return ParseSql( string sql, bool expectErrors, out StringBuilder sb, out CommonTokenStream tokensOutput, Version version) { sql = sql.TrimStart(); MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(sql));//ASCIIEncoding.ASCII.GetBytes(sql)); CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms); //ANTLRInputStream input = new ANTLRInputStream(ms); MySQLLexer lexer = new MySQLLexer(input); lexer.MySqlVersion = version; CommonTokenStream tokens = new CommonTokenStream(lexer); tokensOutput = tokens; return(DoParse(tokens, expectErrors, out sb, lexer.MySqlVersion)); }
private void ParseRQL() { _textMarkerService.RemoveAll(m => true); _errorListener.Reset(); var inputStream = new CaseInsensitiveInputStream(CodeEditor.Text); _lexer.SetInputStream(inputStream); _parser.SetInputStream(new CommonTokenStream(_lexer)); _parser.query(); var errors = _errorListener.Errors.Aggregate(new StringBuilder(), (sb, err) => sb.AppendLine(err.ToString().Replace("\\n", " ") .Replace("\\r", " ") .Replace("\\t", " "))).ToString(); AddSquigglies(); Errors.Text = errors; }
/// <summary> /// Parse the given input into a description of a sequence diagram. /// </summary> /// <param name="input">The input string</param> /// <returns></returns> public static (SequenceDiagramViewModel SequenceDiagram, string Error) Parse(string input) { var inputStream = new CaseInsensitiveInputStream(input); var lexer = new SequenceLexer(inputStream); var tokenStream = new CommonTokenStream(lexer); var parser = new SequenceParser(tokenStream); // Report errors to the console (for now) var error = new StringBuilder(); var errorListener = new StringErrorListener(error); lexer.AddErrorListener(errorListener); parser.AddErrorListener(errorListener); // Generate SequenceDiagramViewModel out of parsed result StartContext startContext = parser.start(); SequenceVisitor visitor = new SequenceVisitor(errorListener); visitor.Visit(startContext); return(visitor.SequenceDiagram, error.ToString()); }
private static StartEventPropertyRuleContext HandleRecognitionEx( RecognitionException e, CommonTokenStream tokens, string propertyName, EsperEPL2GrammarParser g) { // Check for keywords and escape each, parse again var escapedPropertyName = EscapeKeywords(tokens); var inputEscaped = new CaseInsensitiveInputStream(escapedPropertyName); var lexEscaped = ParseHelper.NewLexer(inputEscaped); var tokensEscaped = new CommonTokenStream(lexEscaped); var gEscaped = ParseHelper.NewParser(tokensEscaped); try { return(gEscaped.startEventPropertyRule()); } catch { // ignored } throw ExceptionConvertor.ConvertProperty(e, propertyName, true, g); }
public void TestTokenRemove() { string sql = "select *, a, c, d from table1 where a is null"; MemoryStream ms = new MemoryStream(ASCIIEncoding.ASCII.GetBytes(sql)); CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms); MySQLLexer lexer = new MySQLLexer(input); Parser.TokenStreamRemovable tsr = new Parser.TokenStreamRemovable(lexer); tsr.Fill(); List <IToken> tokens = tsr.GetTokens(); IToken removed = null; foreach (IToken t in tokens) { if (t.Text == "d") { removed = t; break; } } tsr.Remove(removed); tokens = tsr.GetTokens(); }
public static StartEventPropertyRuleContext Parse(string propertyName) { var input = new CaseInsensitiveInputStream(propertyName); var lex = ParseHelper.NewLexer(input); var tokens = new CommonTokenStream(lex); try { tokens.Fill(); } catch (Exception e) { if (ParseHelper.HasControlCharacters(propertyName)) { throw new PropertyAccessException("Unrecognized control characters found in text"); } throw new PropertyAccessException("Failed to parse text: " + e.Message); } var g = ParseHelper.NewParser(tokens); StartEventPropertyRuleContext r; try { r = g.startEventPropertyRule(); } catch (RecognitionException e) { return(HandleRecognitionEx(e, tokens, propertyName, g)); } catch (Exception e) { if (e.InnerException is RecognitionException re) { return(HandleRecognitionEx(re, tokens, propertyName, g)); } else { throw; } } return(r); }
public static IList<EPLModuleParseItem> Parse(string module) { var input = new CaseInsensitiveInputStream(module); var lex = ParseHelper.NewLexer(input); var tokens = new CommonTokenStream(lex); try { tokens.Fill(); } catch (Exception ex) { var message = "Unexpected exception recognizing module text"; if (ex is LexerNoViableAltException) { if (ParseHelper.HasControlCharacters(module)) { message = "Unrecognized control characters found in text, failed to parse text"; } else { message += ", recognition failed for " + ex; } } else if (ex is RecognitionException) { var recog = (RecognitionException) ex; message += ", recognition failed for " + recog; } else if (ex.Message != null) { message += ": " + ex.Message; } message += " [" + module + "]"; Log.Error(message, ex); throw new ParseException(message); } IList<EPLModuleParseItem> statements = new List<EPLModuleParseItem>(); var current = new StringWriter(); int? lineNum = null; var charPosStart = 0; var charPos = 0; var tokenList = tokens.GetTokens(); var skippedSemicolonIndexes = GetSkippedSemicolons(tokenList); var index = -1; // Call getTokens first before invoking tokens.size! ANTLR problem foreach (var token in tokenList) { index++; var t = token; var semi = t.Type == EsperEPL2GrammarLexer.SEMI && !skippedSemicolonIndexes.Contains(index); if (semi) { if (current.ToString().Trim().Length > 0) { statements.Add( new EPLModuleParseItem(current.ToString().Trim(), lineNum ?? 0, charPosStart, charPos)); lineNum = null; } current = new StringWriter(); } else { if (lineNum == null && t.Type != EsperEPL2GrammarParser.WS) { lineNum = t.Line; charPosStart = charPos; } if (t.Type != EsperEPL2GrammarLexer.Eof) { current.Write(t.Text); charPos += t.Text.Length; } } } if (current.ToString().Trim().Length > 0) { statements.Add(new EPLModuleParseItem(current.ToString().Trim(), lineNum ?? 0, 0, 0)); } return statements; }
public static ParseNode GetModule( EPLModuleParseItem item, string resourceName) { ICharStream input = new CaseInsensitiveInputStream(item.Expression); var lex = ParseHelper.NewLexer(input); var tokenStream = new CommonTokenStream(lex); tokenStream.Fill(); var tokens = tokenStream.GetTokens(); var beginIndex = 0; var isMeta = false; var isModule = false; var isUses = false; var isExpression = false; while (beginIndex < tokens.Count) { var t = tokens[beginIndex]; if (t.Type == EsperEPL2GrammarParser.Eof) { break; } if (t.Type == EsperEPL2GrammarParser.WS || t.Type == EsperEPL2GrammarParser.SL_COMMENT || t.Type == EsperEPL2GrammarParser.ML_COMMENT) { beginIndex++; continue; } var tokenText = t.Text.Trim().ToLowerInvariant(); if (tokenText.Equals("module")) { isModule = true; isMeta = true; } else if (tokenText.Equals("uses")) { isUses = true; isMeta = true; } else if (tokenText.Equals("import")) { isMeta = true; } else { isExpression = true; break; } beginIndex++; beginIndex++; // skip space break; } if (isExpression) { return new ParseNodeExpression(item); } if (!isMeta) { return new ParseNodeComment(item); } // check meta tag (module, uses, import) var buffer = new StringWriter(); for (var i = beginIndex; i < tokens.Count; i++) { var t = tokens[i]; if (t.Type == EsperEPL2GrammarParser.Eof) { break; } if (t.Type != EsperEPL2GrammarParser.IDENT && t.Type != EsperEPL2GrammarParser.DOT && t.Type != EsperEPL2GrammarParser.STAR && !t.Text.Matches("[a-zA-Z]*")) { throw GetMessage(isModule, isUses, resourceName, t.Type); } buffer.Write(t.Text.Trim()); } var result = buffer.ToString().Trim(); if (result.Length == 0) { throw GetMessage(isModule, isUses, resourceName, -1); } if (isModule) { return new ParseNodeModule(item, result); } if (isUses) { return new ParseNodeUses(item, result); } return new ParseNodeImport(item, result); }
/// <summary> /// Lexes the sample SQL and inserts a "where 1=0" where-clause. /// </summary> /// <param name="querySQL">to inspect using lexer</param> /// <returns>sample SQL with where-clause inserted</returns> /// <throws>ExprValidationException to indicate a lexer problem</throws> public static string LexSampleSQL(string querySQL) { querySQL = querySQL.RegexReplaceAll("\\s\\s+|\\n|\\r", " "); var input = new CaseInsensitiveInputStream(querySQL); var whereIndex = -1; var groupbyIndex = -1; var havingIndex = -1; var orderByIndex = -1; IList<int> unionIndexes = new List<int>(); var lex = ParseHelper.NewLexer(input); var tokens = new CommonTokenStream(lex); tokens.Fill(); var tokenList = tokens.GetTokens(); for (var i = 0; i < tokenList.Count; i++) { var token = tokenList[i]; if ((token == null) || token.Text == null) { break; } var text = token.Text.ToLowerInvariant().Trim(); if (text.Equals("where")) { whereIndex = token.Column + 1; } if (text.Equals("group")) { groupbyIndex = token.Column + 1; } if (text.Equals("having")) { havingIndex = token.Column + 1; } if (text.Equals("order")) { orderByIndex = token.Column + 1; } if (text.Equals("union")) { unionIndexes.Add(token.Column + 1); } } // If we have a union, break string into subselects and process each if (unionIndexes.Count != 0) { var changedSQL = new StringWriter(); var lastIndex = 0; for (var i = 0; i < unionIndexes.Count; i++) { var index = unionIndexes[i]; string fragmentX; if (i > 0) { fragmentX = querySQL.Between(lastIndex + 5, index - 1); } else { fragmentX = querySQL.Between(lastIndex, index - 1); } var lexedFragmentX = LexSampleSQL(fragmentX); if (i > 0) { changedSQL.Write("union "); } changedSQL.Write(lexedFragmentX); lastIndex = index - 1; } // last part after last union var fragment = querySQL.Substring(lastIndex + 5); var lexedFragment = LexSampleSQL(fragment); changedSQL.Write("union "); changedSQL.Write(lexedFragment); return changedSQL.ToString(); } // Found a where clause, simplest cases if (whereIndex != -1) { var changedSQL = new StringWriter(); var prefix = querySQL.Substring(0, whereIndex + 5); var suffix = querySQL.Substring(whereIndex + 5); changedSQL.Write(prefix); changedSQL.Write("1=0 and "); changedSQL.Write(suffix); return changedSQL.ToString(); } // No where clause, find group-by int insertIndex; if (groupbyIndex != -1) { insertIndex = groupbyIndex; } else if (havingIndex != -1) { insertIndex = havingIndex; } else if (orderByIndex != -1) { insertIndex = orderByIndex; } else { var changedSQL = new StringWriter(); changedSQL.Write(querySQL); changedSQL.Write(" where 1=0 "); return changedSQL.ToString(); } try { var changedSQL = new StringWriter(); var prefix = querySQL.Substring(0, insertIndex - 1); changedSQL.Write(prefix); changedSQL.Write("where 1=0 "); var suffix = querySQL.Substring(insertIndex - 1); changedSQL.Write(suffix); return changedSQL.ToString(); } catch (Exception ex) { var text = "Error constructing sample SQL to retrieve metadata for ADO-drivers that don't support metadata, consider using the " + SAMPLE_WHERECLAUSE_PLACEHOLDER + " placeholder or providing a sample SQL"; Log.Error(text, ex); throw new ExprValidationException(text, ex); } }
/// <summary> /// Parse expression using the rule the ParseRuleSelector instance supplies. /// </summary> /// <param name="expression">text to parse</param> /// <param name="parseRuleSelector">parse rule to select</param> /// <param name="addPleaseCheck">true to include depth paraphrase</param> /// <param name="eplStatementErrorMsg">text for error</param> /// <param name="rewriteScript">whether to rewrite script expressions</param> /// <returns>AST - syntax tree</returns> /// <throws>EPException when the AST could not be parsed</throws> /// <throws>StatementSpecCompileSyntaxException syntax exceptions</throws> public static ParseResult Parse( string expression, string eplStatementErrorMsg, bool addPleaseCheck, ParseRuleSelector parseRuleSelector, bool rewriteScript) { if (Log.IsDebugEnabled) { Log.Debug(".parse Parsing expr=" + expression); } var input = new CaseInsensitiveInputStream(expression); var lex = NewLexer(input); var tokens = new CommonTokenStream(lex); var parser = ParseHelper.NewParser(tokens); ITree tree; try { tree = parseRuleSelector.InvokeParseRule(parser); } catch (RecognitionException ex) { tokens.Fill(); if (rewriteScript) { if (IsContainsScriptOrClassExpression(tokens)) { return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector); } } Log.Debug("Error parsing statement [" + expression + "]", ex); throw ExceptionConvertor.ConvertStatement(ex, eplStatementErrorMsg, addPleaseCheck, parser); } catch (Exception e) { try { tokens.Fill(); } catch (Exception) { Log.Debug("Token-fill produced exception: " + e.Message, e); } if (Log.IsDebugEnabled) { Log.Debug("Error parsing statement [" + eplStatementErrorMsg + "]", e); } if (e.InnerException is RecognitionException recognitionException) { if (rewriteScript) { if (IsContainsScriptOrClassExpression(tokens)) { return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector); } } throw ExceptionConvertor.ConvertStatement(recognitionException, eplStatementErrorMsg, addPleaseCheck, parser); } throw; } // if we are re-writing scripts and contain a script, then rewrite if (rewriteScript && IsContainsScriptOrClassExpression(tokens)) { return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector); } if (Log.IsDebugEnabled) { Log.Debug(".parse Dumping AST..."); ASTUtil.DumpAST(tree); } var expressionWithoutAnnotation = expression; if (tree is EsperEPL2GrammarParser.StartEPLExpressionRuleContext) { var epl = (EsperEPL2GrammarParser.StartEPLExpressionRuleContext) tree; expressionWithoutAnnotation = GetNoAnnotation(expression, epl.annotationEnum(), tokens); } return new ParseResult( tree, expressionWithoutAnnotation, tokens, EmptyList<string>.Instance, EmptyList<string>.Instance); }