public static MySQL51Parser.program_return ParseSql(string sql, bool expectErrors, out StringBuilder sb)
{
// The grammar supports upper case only
MemoryStream ms = new MemoryStream(ASCIIEncoding.ASCII.GetBytes(sql/*.ToUpper() */));
CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms);
//ANTLRInputStream input = new ANTLRInputStream(ms);
MySQL51Lexer lexer = new MySQL51Lexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
MySQL51Parser parser = new MySQL51Parser(tokens);
sb = new StringBuilder();
TextWriter tw = new StringWriter(sb);
parser.TraceDestination = tw;
MySQL51Parser.program_return r = parser.program();
if (!expectErrors)
{
if (0 != parser.NumberOfSyntaxErrors)
Assert.AreEqual("", sb.ToString());
//Assert.AreEqual( 0, parser.NumberOfSyntaxErrors);
}
else
{
Assert.AreNotEqual(0, parser.NumberOfSyntaxErrors);
}
return r;
}
public static TokenStreamRemovable GetTokenStream(string sql, Version version)
{
Dictionary <Version, TokenStreamRemovable> lines;
if (_parserCache.TryGetValue(sql, out lines))
{
TokenStreamRemovable tsr;
if (lines.TryGetValue(version, out tsr))
{
return(tsr);
}
}
// no cache entry, then parse
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(sql));//ASCIIEncoding.ASCII.GetBytes(sql));
CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms);
MySQLLexer lexer = new MySQLLexer(input);
lexer.MySqlVersion = version;
TokenStreamRemovable tokens = new TokenStreamRemovable(lexer);
if (lines == null)
{
lines = new Dictionary <Version, TokenStreamRemovable>();
_parserCache.Add(sql, lines);
}
lines.Add(version, tokens);
return(tokens);
}
public void CaseInsensitiveLexerTest()
{
var stream = new CaseInsensitiveInputStream(data);
var lexer = new CaseInsensitiveLexer(stream);
lexer.AddErrorListener(errorListener);
var tokens = lexer.GetAllTokens();
}
internal static MySQL51Parser.program_return ParseSql(
string sql, bool expectErrors, out StringBuilder sb, out CommonTokenStream tokensOutput, Version version)
{
sql = sql.TrimStart();
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(sql));//ASCIIEncoding.ASCII.GetBytes(sql));
CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms);
//ANTLRInputStream input = new ANTLRInputStream(ms);
MySQLLexer lexer = new MySQLLexer(input);
lexer.MySqlVersion = version;
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokensOutput = tokens;
return(DoParse(tokens, expectErrors, out sb, lexer.MySqlVersion));
}
private void ParseRQL()
{
_textMarkerService.RemoveAll(m => true);
_errorListener.Reset();
var inputStream = new CaseInsensitiveInputStream(CodeEditor.Text);
_lexer.SetInputStream(inputStream);
_parser.SetInputStream(new CommonTokenStream(_lexer));
_parser.query();
var errors = _errorListener.Errors.Aggregate(new StringBuilder(), (sb, err) =>
sb.AppendLine(err.ToString().Replace("\\n", " ")
.Replace("\\r", " ")
.Replace("\\t", " "))).ToString();
AddSquigglies();
Errors.Text = errors;
}
/// <summary>
/// Parse the given input into a description of a sequence diagram.
/// </summary>
/// <param name="input">The input string</param>
/// <returns></returns>
public static (SequenceDiagramViewModel SequenceDiagram, string Error) Parse(string input)
{
var inputStream = new CaseInsensitiveInputStream(input);
var lexer = new SequenceLexer(inputStream);
var tokenStream = new CommonTokenStream(lexer);
var parser = new SequenceParser(tokenStream);
// Report errors to the console (for now)
var error = new StringBuilder();
var errorListener = new StringErrorListener(error);
lexer.AddErrorListener(errorListener);
parser.AddErrorListener(errorListener);
// Generate SequenceDiagramViewModel out of parsed result
StartContext startContext = parser.start();
SequenceVisitor visitor = new SequenceVisitor(errorListener);
visitor.Visit(startContext);
return(visitor.SequenceDiagram, error.ToString());
}
private static StartEventPropertyRuleContext HandleRecognitionEx(
RecognitionException e,
CommonTokenStream tokens,
string propertyName,
EsperEPL2GrammarParser g)
{
// Check for keywords and escape each, parse again
var escapedPropertyName = EscapeKeywords(tokens);
var inputEscaped = new CaseInsensitiveInputStream(escapedPropertyName);
var lexEscaped = ParseHelper.NewLexer(inputEscaped);
var tokensEscaped = new CommonTokenStream(lexEscaped);
var gEscaped = ParseHelper.NewParser(tokensEscaped);
try {
return(gEscaped.startEventPropertyRule());
}
catch {
// ignored
}
throw ExceptionConvertor.ConvertProperty(e, propertyName, true, g);
}
public void TestTokenRemove()
{
string sql = "select *, a, c, d from table1 where a is null";
MemoryStream ms = new MemoryStream(ASCIIEncoding.ASCII.GetBytes(sql));
CaseInsensitiveInputStream input = new CaseInsensitiveInputStream(ms);
MySQLLexer lexer = new MySQLLexer(input);
Parser.TokenStreamRemovable tsr = new Parser.TokenStreamRemovable(lexer);
tsr.Fill();
List <IToken> tokens = tsr.GetTokens();
IToken removed = null;
foreach (IToken t in tokens)
{
if (t.Text == "d")
{
removed = t;
break;
}
}
tsr.Remove(removed);
tokens = tsr.GetTokens();
}
public static StartEventPropertyRuleContext Parse(string propertyName)
{
var input = new CaseInsensitiveInputStream(propertyName);
var lex = ParseHelper.NewLexer(input);
var tokens = new CommonTokenStream(lex);
try {
tokens.Fill();
} catch (Exception e) {
if (ParseHelper.HasControlCharacters(propertyName))
{
throw new PropertyAccessException("Unrecognized control characters found in text");
}
throw new PropertyAccessException("Failed to parse text: " + e.Message);
}
var g = ParseHelper.NewParser(tokens);
StartEventPropertyRuleContext r;
try {
r = g.startEventPropertyRule();
} catch (RecognitionException e) {
return(HandleRecognitionEx(e, tokens, propertyName, g));
} catch (Exception e) {
if (e.InnerException is RecognitionException re)
{
return(HandleRecognitionEx(re, tokens, propertyName, g));
}
else
{
throw;
}
}
return(r);
}
public static IList<EPLModuleParseItem> Parse(string module)
{
var input = new CaseInsensitiveInputStream(module);
var lex = ParseHelper.NewLexer(input);
var tokens = new CommonTokenStream(lex);
try {
tokens.Fill();
}
catch (Exception ex) {
var message = "Unexpected exception recognizing module text";
if (ex is LexerNoViableAltException) {
if (ParseHelper.HasControlCharacters(module)) {
message = "Unrecognized control characters found in text, failed to parse text";
}
else {
message += ", recognition failed for " + ex;
}
}
else if (ex is RecognitionException) {
var recog = (RecognitionException) ex;
message += ", recognition failed for " + recog;
}
else if (ex.Message != null) {
message += ": " + ex.Message;
}
message += " [" + module + "]";
Log.Error(message, ex);
throw new ParseException(message);
}
IList<EPLModuleParseItem> statements = new List<EPLModuleParseItem>();
var current = new StringWriter();
int? lineNum = null;
var charPosStart = 0;
var charPos = 0;
var tokenList = tokens.GetTokens();
var skippedSemicolonIndexes = GetSkippedSemicolons(tokenList);
var index = -1;
// Call getTokens first before invoking tokens.size! ANTLR problem
foreach (var token in tokenList) {
index++;
var t = token;
var semi = t.Type == EsperEPL2GrammarLexer.SEMI && !skippedSemicolonIndexes.Contains(index);
if (semi) {
if (current.ToString().Trim().Length > 0) {
statements.Add(
new EPLModuleParseItem(current.ToString().Trim(), lineNum ?? 0, charPosStart, charPos));
lineNum = null;
}
current = new StringWriter();
}
else {
if (lineNum == null && t.Type != EsperEPL2GrammarParser.WS) {
lineNum = t.Line;
charPosStart = charPos;
}
if (t.Type != EsperEPL2GrammarLexer.Eof) {
current.Write(t.Text);
charPos += t.Text.Length;
}
}
}
if (current.ToString().Trim().Length > 0) {
statements.Add(new EPLModuleParseItem(current.ToString().Trim(), lineNum ?? 0, 0, 0));
}
return statements;
}
public static ParseNode GetModule(
EPLModuleParseItem item,
string resourceName)
{
ICharStream input = new CaseInsensitiveInputStream(item.Expression);
var lex = ParseHelper.NewLexer(input);
var tokenStream = new CommonTokenStream(lex);
tokenStream.Fill();
var tokens = tokenStream.GetTokens();
var beginIndex = 0;
var isMeta = false;
var isModule = false;
var isUses = false;
var isExpression = false;
while (beginIndex < tokens.Count) {
var t = tokens[beginIndex];
if (t.Type == EsperEPL2GrammarParser.Eof) {
break;
}
if (t.Type == EsperEPL2GrammarParser.WS ||
t.Type == EsperEPL2GrammarParser.SL_COMMENT ||
t.Type == EsperEPL2GrammarParser.ML_COMMENT) {
beginIndex++;
continue;
}
var tokenText = t.Text.Trim().ToLowerInvariant();
if (tokenText.Equals("module")) {
isModule = true;
isMeta = true;
}
else if (tokenText.Equals("uses")) {
isUses = true;
isMeta = true;
}
else if (tokenText.Equals("import")) {
isMeta = true;
}
else {
isExpression = true;
break;
}
beginIndex++;
beginIndex++; // skip space
break;
}
if (isExpression) {
return new ParseNodeExpression(item);
}
if (!isMeta) {
return new ParseNodeComment(item);
}
// check meta tag (module, uses, import)
var buffer = new StringWriter();
for (var i = beginIndex; i < tokens.Count; i++) {
var t = tokens[i];
if (t.Type == EsperEPL2GrammarParser.Eof) {
break;
}
if (t.Type != EsperEPL2GrammarParser.IDENT &&
t.Type != EsperEPL2GrammarParser.DOT &&
t.Type != EsperEPL2GrammarParser.STAR &&
!t.Text.Matches("[a-zA-Z]*")) {
throw GetMessage(isModule, isUses, resourceName, t.Type);
}
buffer.Write(t.Text.Trim());
}
var result = buffer.ToString().Trim();
if (result.Length == 0) {
throw GetMessage(isModule, isUses, resourceName, -1);
}
if (isModule) {
return new ParseNodeModule(item, result);
}
if (isUses) {
return new ParseNodeUses(item, result);
}
return new ParseNodeImport(item, result);
}
/// <summary>
/// Lexes the sample SQL and inserts a "where 1=0" where-clause.
/// </summary>
/// <param name="querySQL">to inspect using lexer</param>
/// <returns>sample SQL with where-clause inserted</returns>
/// <throws>ExprValidationException to indicate a lexer problem</throws>
public static string LexSampleSQL(string querySQL)
{
querySQL = querySQL.RegexReplaceAll("\\s\\s+|\\n|\\r", " ");
var input = new CaseInsensitiveInputStream(querySQL);
var whereIndex = -1;
var groupbyIndex = -1;
var havingIndex = -1;
var orderByIndex = -1;
IList<int> unionIndexes = new List<int>();
var lex = ParseHelper.NewLexer(input);
var tokens = new CommonTokenStream(lex);
tokens.Fill();
var tokenList = tokens.GetTokens();
for (var i = 0; i < tokenList.Count; i++) {
var token = tokenList[i];
if ((token == null) || token.Text == null) {
break;
}
var text = token.Text.ToLowerInvariant().Trim();
if (text.Equals("where")) {
whereIndex = token.Column + 1;
}
if (text.Equals("group")) {
groupbyIndex = token.Column + 1;
}
if (text.Equals("having")) {
havingIndex = token.Column + 1;
}
if (text.Equals("order")) {
orderByIndex = token.Column + 1;
}
if (text.Equals("union")) {
unionIndexes.Add(token.Column + 1);
}
}
// If we have a union, break string into subselects and process each
if (unionIndexes.Count != 0) {
var changedSQL = new StringWriter();
var lastIndex = 0;
for (var i = 0; i < unionIndexes.Count; i++) {
var index = unionIndexes[i];
string fragmentX;
if (i > 0) {
fragmentX = querySQL.Between(lastIndex + 5, index - 1);
}
else {
fragmentX = querySQL.Between(lastIndex, index - 1);
}
var lexedFragmentX = LexSampleSQL(fragmentX);
if (i > 0) {
changedSQL.Write("union ");
}
changedSQL.Write(lexedFragmentX);
lastIndex = index - 1;
}
// last part after last union
var fragment = querySQL.Substring(lastIndex + 5);
var lexedFragment = LexSampleSQL(fragment);
changedSQL.Write("union ");
changedSQL.Write(lexedFragment);
return changedSQL.ToString();
}
// Found a where clause, simplest cases
if (whereIndex != -1) {
var changedSQL = new StringWriter();
var prefix = querySQL.Substring(0, whereIndex + 5);
var suffix = querySQL.Substring(whereIndex + 5);
changedSQL.Write(prefix);
changedSQL.Write("1=0 and ");
changedSQL.Write(suffix);
return changedSQL.ToString();
}
// No where clause, find group-by
int insertIndex;
if (groupbyIndex != -1) {
insertIndex = groupbyIndex;
}
else if (havingIndex != -1) {
insertIndex = havingIndex;
}
else if (orderByIndex != -1) {
insertIndex = orderByIndex;
}
else {
var changedSQL = new StringWriter();
changedSQL.Write(querySQL);
changedSQL.Write(" where 1=0 ");
return changedSQL.ToString();
}
try {
var changedSQL = new StringWriter();
var prefix = querySQL.Substring(0, insertIndex - 1);
changedSQL.Write(prefix);
changedSQL.Write("where 1=0 ");
var suffix = querySQL.Substring(insertIndex - 1);
changedSQL.Write(suffix);
return changedSQL.ToString();
}
catch (Exception ex) {
var text =
"Error constructing sample SQL to retrieve metadata for ADO-drivers that don't support metadata, consider using the " +
SAMPLE_WHERECLAUSE_PLACEHOLDER +
" placeholder or providing a sample SQL";
Log.Error(text, ex);
throw new ExprValidationException(text, ex);
}
}
/// <summary>
/// Parse expression using the rule the ParseRuleSelector instance supplies.
/// </summary>
/// <param name="expression">text to parse</param>
/// <param name="parseRuleSelector">parse rule to select</param>
/// <param name="addPleaseCheck">true to include depth paraphrase</param>
/// <param name="eplStatementErrorMsg">text for error</param>
/// <param name="rewriteScript">whether to rewrite script expressions</param>
/// <returns>AST - syntax tree</returns>
/// <throws>EPException when the AST could not be parsed</throws>
/// <throws>StatementSpecCompileSyntaxException syntax exceptions</throws>
public static ParseResult Parse(
string expression,
string eplStatementErrorMsg,
bool addPleaseCheck,
ParseRuleSelector parseRuleSelector,
bool rewriteScript)
{
if (Log.IsDebugEnabled)
{
Log.Debug(".parse Parsing expr=" + expression);
}
var input = new CaseInsensitiveInputStream(expression);
var lex = NewLexer(input);
var tokens = new CommonTokenStream(lex);
var parser = ParseHelper.NewParser(tokens);
ITree tree;
try
{
tree = parseRuleSelector.InvokeParseRule(parser);
}
catch (RecognitionException ex)
{
tokens.Fill();
if (rewriteScript)
{
if (IsContainsScriptOrClassExpression(tokens))
{
return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector);
}
}
Log.Debug("Error parsing statement [" + expression + "]", ex);
throw ExceptionConvertor.ConvertStatement(ex, eplStatementErrorMsg, addPleaseCheck, parser);
}
catch (Exception e)
{
try
{
tokens.Fill();
}
catch (Exception)
{
Log.Debug("Token-fill produced exception: " + e.Message, e);
}
if (Log.IsDebugEnabled)
{
Log.Debug("Error parsing statement [" + eplStatementErrorMsg + "]", e);
}
if (e.InnerException is RecognitionException recognitionException)
{
if (rewriteScript)
{
if (IsContainsScriptOrClassExpression(tokens))
{
return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector);
}
}
throw ExceptionConvertor.ConvertStatement(recognitionException, eplStatementErrorMsg, addPleaseCheck, parser);
}
throw;
}
// if we are re-writing scripts and contain a script, then rewrite
if (rewriteScript && IsContainsScriptOrClassExpression(tokens))
{
return HandleScriptAndClassRewrite(tokens, eplStatementErrorMsg, addPleaseCheck, parseRuleSelector);
}
if (Log.IsDebugEnabled)
{
Log.Debug(".parse Dumping AST...");
ASTUtil.DumpAST(tree);
}
var expressionWithoutAnnotation = expression;
if (tree is EsperEPL2GrammarParser.StartEPLExpressionRuleContext)
{
var epl = (EsperEPL2GrammarParser.StartEPLExpressionRuleContext) tree;
expressionWithoutAnnotation = GetNoAnnotation(expression, epl.annotationEnum(), tokens);
}
return new ParseResult(
tree,
expressionWithoutAnnotation,
tokens,
EmptyList<string>.Instance,
EmptyList<string>.Instance);
}
