public KustoQuickInfoBuilder(KustoCodeService service, KustoCode code, QuickInfoOptions options)
{
_service = service;
_code = code;
_options = options;
_disabled = new DisabledDiagnostics(_options.DisabledDiagnostics);
}
/// <summary>
/// Loads and adds the <see cref="DatabaseSymbol"/> for any database explicity referenced in the query but not already present in the <see cref="GlobalState"/>.
/// </summary>
public async Task <KustoCode> AddReferencedDatabasesAsync(KustoCode code, CancellationToken cancellationToken = default)
{
var service = new KustoCodeService(code);
var globals = await AddReferencedDatabasesAsync(code.Globals, service, cancellationToken).ConfigureAwait(false);
return(code.WithGlobals(globals));
}
/// <summary>
/// Translate Kusto parameter list declaration into into list of <see cref="Parameter"/> instances.
/// </summary>
private static IReadOnlyList <Parameter> TranslateParameters(string parameters)
{
parameters = parameters.Trim();
if (string.IsNullOrEmpty(parameters) || parameters == "()")
{
return(NoParameters);
}
if (parameters[0] != '(')
{
parameters = "(" + parameters;
}
if (parameters[parameters.Length - 1] != ')')
{
parameters = parameters + ")";
}
var query = "let fn = " + parameters + " { };";
var code = KustoCode.ParseAndAnalyze(query);
var let = code.Syntax.GetFirstDescendant <LetStatement>();
if (let.Name.ReferencedSymbol is FunctionSymbol fs)
{
return(fs.Signatures[0].Parameters);
}
else if (let.Name.ReferencedSymbol is VariableSymbol vs &&
vs.Type is FunctionSymbol vfs)
{
return(vfs.Signatures[0].Parameters);
}
private List <RxKqlScalarValue> ParseExpressionKusto(string extend)
{
extend = extend.Trim();
if (!extend.StartsWith("extend "))
{
extend = "extend " + extend;
}
KustoCode query;
lock (parserLock)
{
query = KustoCode.Parse(extend);
}
var diagnostics = query.GetSyntaxDiagnostics()
.Select(d => $"({d.Start}..{d.Start + d.Length}): {d.Message}");
if (diagnostics.Any())
{
var errors = string.Join("\n", diagnostics);
throw new QueryParsingException($"Error parsing expression {extend}: {errors}");
}
var syntax = query.Syntax.GetDescendants <Statement>()[0];
return(syntax.Visit(new ListRxKqlScalarValueConverter()));
}
private static string GetOutlineCollapsedText(KustoCode code)
{
var builder = new StringBuilder();
for (int i = 0; i < code.LexerTokens.Count; i++)
{
var token = code.LexerTokens[i];
if (token.Text == "|" || token.Text == ";")
{
break;
}
if (token.Trivia.Length > 0)
{
if (i == 0)
{
builder.Append(token.Trivia);
}
else
{
builder.Append(" ");
}
}
builder.Append(token.Text);
}
return(builder.ToString());
}
private static string GetOutlineCollapsedText(KustoCode code)
{
var builder = new StringBuilder();
for (var token = code.Syntax.GetFirstToken(); token != null; token = token.GetNextToken())
{
if (token.Text == "|" || token.Text == ";")
{
break;
}
if (token.Trivia.Length > 0)
{
if (builder.Length == 0)
{
builder.Append(token.Trivia);
}
else
{
builder.Append(" ");
}
}
builder.Append(token.Text);
}
return(builder.ToString());
}
public IReadOnlyList <RuleOutcome> Analyze(KustoCode code)
{
var outcomes = new List <RuleOutcome>();
foreach (var node in code.Syntax.GetDescendants <BinaryExpression>())
{
if (node.Kind == SyntaxKind.ContainsExpression ||
node.Kind == SyntaxKind.NotContainsExpression ||
node.Kind == SyntaxKind.ContainsCsExpression ||
node.Kind == SyntaxKind.NotContainsCsExpression)
{
var ruleOutcome = new RuleOutcome(Name,
score: 10,
message:
$"Avoid using 'contains' operator as it has high compute price." + Environment.NewLine +
$"Use 'has' operator in cases when full term match is desired.",
referenceText: node.ToString(),
severity: Severity.Suggestion,
category: Category.Performance,
textStart: node.TextStart);
outcomes.Add(ruleOutcome);
}
}
return(outcomes);
}
public override void Analyze(KustoCode code, List<Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants<BinaryExpression>())
{
if (!node.Right.IsConstant && !node.Left.IsConstant)
{
continue;
}
if (node.Kind == SyntaxKind.EqualExpression ||
node.Kind == SyntaxKind.HasExpression ||
node.Kind == SyntaxKind.NotEqualExpression ||
node.Kind == SyntaxKind.NotHasExpression ||
node.Kind == SyntaxKind.StartsWithExpression ||
node.Kind == SyntaxKind.NotStartsWithExpression)
{
string constValue = null;
if (node.Right.IsConstant && node.Right.ResultType == ScalarTypes.String)
{
constValue = node.Right.ConstantValue as string;
}
else if (node.Left.IsConstant && node.Left.ResultType == ScalarTypes.String)
{
constValue = node.Left.ConstantValue as string;
}
if (!string.IsNullOrEmpty(constValue) && constValue.Length < 4)
{
diagnostics.Add(_diagnostic.WithLocation(node));
}
}
}
}
public async Task TestAddReferencedDatabasesAsync_KustoCode()
{
var loader = new SymbolLoader(HelpConnection);
// set default database to database other than Samples.
var globals = await loader.AddOrUpdateDefaultDatabaseAsync(GlobalState.Default, "KustoMonitoringPersistentDatabase");
// just one database should exist
Assert.AreEqual(1, globals.Cluster.Databases.Count);
// parse query that has explicit reference to Samples database.
var code = KustoCode.ParseAndAnalyze("database('Samples').StormEvents", globals);
// use loader to add symbols for any explicity referenced databases
var newCode = await loader.AddReferencedDatabasesAsync(code);
// both databases should exist now
Assert.AreEqual(2, newCode.Globals.Cluster.Databases.Count);
// find StormEvents table in Samples database
var samples = newCode.Globals.Cluster.Databases.First(db => db.Name == "Samples");
var storm = samples.Members.First(m => m.Name == "StormEvents");
// verify that query expression returns StormEvents table
var qb = (QueryBlock)newCode.Syntax;
var expr = (qb.Statements[qb.Statements.Count - 1].Element as ExpressionStatement).Expression;
Assert.AreSame(storm, expr.ResultType);
}
/// <summary>
/// Gets the <see cref="KustoCode"/> for the text without waiting for semantic analysis.
/// </summary>
private bool TryGetBoundOrUnboundCode(CancellationToken cancellationToken, bool waitForAnalysis, out KustoCode code)
{
if (this.lazyUnboundCode == null &&
this.codeException == null &&
waitForAnalysis &&
CanBeParsed(this.Text))
{
lock (this) // don't let multiple threads duplicate computation work
{
try
{
if (this.lazyBoundCode != null)
{
// if bound code is already available, use it instead
code = this.lazyBoundCode;
return(true);
}
else
{
var newCode = KustoCode.Parse(this.Text, this.globals);
Interlocked.CompareExchange(ref this.lazyUnboundCode, newCode, null);
}
}
catch (Exception e)
{
this.codeException = e;
}
}
}
code = this.lazyUnboundCode;
return(code != null);
}
private ScalarValue ParseExpressionKusto(string summarize)
{
summarize = summarize.Trim();
if (!summarize.StartsWith("summarize "))
{
summarize = "summarize " + summarize;
}
KustoCode query;
lock (parserLock)
{
query = KustoCode.Parse(summarize);
}
var diagnostics = query.GetSyntaxDiagnostics()
.Select(d => $"({d.Start}..{d.Start + d.Length}): {d.Message}");
if (diagnostics.Any())
{
var errors = string.Join("\n", diagnostics);
throw new QueryParsingException($"Error parsing expression {summarize}: {errors}");
}
var syntax = query.Syntax.GetDescendants <Statement>()[0];
return(syntax.Visit(new ScalarValueConverter()));
}
/// <summary>
/// Gets the <see cref="KustoCode"/> for the text with semantic analysis done.
/// </summary>
private bool TryGetBoundCode(CancellationToken cancellationToken, bool waitForAnalysis, out KustoCode code)
{
if (this.lazyBoundCode == null &&
this.codeException == null &&
waitForAnalysis &&
CanBeParsed(this.Text))
{
lock (this) // don't let multiple threads duplicate computation work
{
try
{
if (this.lazyUnboundCode != null)
{
// if unbound code is already avaiable, do semantic analysis on it (faster, no need to retokenize)
var newCode = this.lazyUnboundCode.Analyze(cancellationToken: cancellationToken);
Interlocked.CompareExchange(ref this.lazyBoundCode, newCode, null);
}
else
{
var newCode = KustoCode.ParseAndAnalyze(this.Text, this.globals, cancellationToken: cancellationToken);
Interlocked.CompareExchange(ref this.lazyBoundCode, newCode, null);
}
}
catch (Exception e)
{
this.codeException = e;
}
}
}
code = this.lazyBoundCode;
return(code != null);
}
private List <RxKqlScalarValue> ParseExpressionKusto(string projectkeep)
{
projectkeep = projectkeep.Trim();
if (!projectkeep.StartsWith("project-keep "))
{
projectkeep = "project-keep " + projectkeep;
}
KustoCode query;
lock (parserLock)
{
query = KustoCode.Parse(projectkeep);
}
var diagnostics = query.GetSyntaxDiagnostics()
.Select(d => $"({d.Start}..{d.Start + d.Length}): {d.Message}");
if (diagnostics.Any())
{
var errors = string.Join("\n", diagnostics);
throw new QueryParsingException($"Error parsing expression {projectkeep}: {errors}");
}
var syntax = query.Syntax.GetDescendants <Statement>()[0];
return(syntax.Accept(new ListRxKqlScalarValueConverter()));
}
public static IObservable <IDictionary <string, object> > KustoQuery(this IObservable <IDictionary <string, object> > source, string query)
{
var kq = KustoCode.Parse(query);
IDictionary <string, object> letValues = new Dictionary <string, object>();
if (kq.Syntax.GetDescendants <Statement>().Count > 1)
{
var statementList = kq.Syntax.GetDescendants <Statement>().ToList();
var letStatements = statementList.Where(x => x.Kind == SyntaxKind.LetStatement);
var queryStatement = statementList.FirstOrDefault(x => x.Kind == SyntaxKind.ExpressionStatement);
}
var lexicalTokens = KustoLexer.GetTokens(query, alwaysProduceEOF: true);
string[] pipeline = SplitExpressions(lexicalTokens).ToArray();
var result = source;
foreach (string p in pipeline)
{
string stage = p.Trim();
int index = stage.IndexOf(' ');
string op = stage.Substring(0, index);
string args = stage.Substring(index + 1);
switch (op)
{
case "where":
result = result.Where(args);
break;
case "limit":
result = result.Take(int.Parse(args));
break;
case "project":
result = result.ProjectExpressions(args);
break;
case "evaluate":
result = result.Evaluate(args);
break;
case "extend":
result = result.Extend(args);
break;
case "summarize":
result = result.Summarize(args);
break;
default:
throw new NotImplementedException($"KustoQuery observable does not implement the operator: {op}");
}
}
return(result);
}
public IReadOnlyList <RuleOutcome> Analyze(KustoCode code)
{
var outcomes = new List <RuleOutcome>();
foreach (var rule in Rules)
{
outcomes.AddRange(rule.Analyze(code));
}
return(outcomes);
}
public override void Analyze(KustoCode code, List <Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants <FunctionCallExpression>(fc => fc.ReferencedSymbol == Functions.FormatDatetime))
{
if (IsInFilter(node) || IsInPredicate(node))
{
diagnostics.Add(_diagnostic.WithLocation(node));
}
}
}
public IReadOnlyList <RuleOutcome> Analyze(KustoCode code)
{
var outcomes = new List <RuleOutcome>();
foreach (var node in code.Syntax.GetDescendants <SummarizeOperator>())
{
var invocations = GetProblematicInvocations(node);
var results = invocations.Select(inv => GetOutcomeRule(inv));
outcomes.AddRange(results);
}
return(outcomes);
}
private KustoCodeService(string text, GlobalState globals, KustoCode code)
: base(text)
{
if (globals == null)
{
throw new ArgumentNullException(nameof(globals));
}
this.kind = KustoCode.GetKind(text);
this.globals = globals;
this.lazyBoundCode = code;
}
public override void Analyze(KustoCode code, List <Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants <BinaryExpression>())
{
if (node.Kind == SyntaxKind.ContainsExpression ||
node.Kind == SyntaxKind.NotContainsExpression ||
node.Kind == SyntaxKind.ContainsCsExpression ||
node.Kind == SyntaxKind.NotContainsCsExpression)
{
diagnostics.Add(_diagnostic.WithLocation(node.Operator));
}
}
}
public IReadOnlyList <RuleOutcome> Analyze(KustoCode code)
{
var outcomes = new List <RuleOutcome>();
foreach (var node in code.Syntax.GetDescendants <BinaryExpression>())
{
if (!node.Right.IsConstant && !node.Left.IsConstant)
{
continue;
}
if (node.Kind == SyntaxKind.EqualExpression ||
node.Kind == SyntaxKind.HasExpression ||
node.Kind == SyntaxKind.NotEqualExpression ||
node.Kind == SyntaxKind.NotHasExpression ||
node.Kind == SyntaxKind.StartsWithExpression ||
node.Kind == SyntaxKind.NotStartsWithExpression)
{
string constValue = null;
if (node.Right.IsConstant && node.Right.ResultType.Name == "string")
{
constValue = node.Right.ConstantValue?.ToString();
}
else if (node.Left.IsConstant && node.Left.ResultType.Name == "string")
{
constValue = node.Left.ConstantValue?.ToString();
}
if (string.IsNullOrEmpty(constValue))
{
continue;
}
if (constValue.Length < 4)
{
var ruleOutcome = new RuleOutcome(Name,
score: 10,
message: Description,
referenceText: node.ToString(),
severity: Severity.Suggestion,
category: Category.Performance,
textStart: node.TextStart);
outcomes.Add(ruleOutcome);
}
}
}
return(outcomes);
}
public override IReadOnlyList <Diagnostic> Analyze(KustoCode code, CancellationToken cancellationToken)
{
var diagnostics = new List <Diagnostic>();
foreach (var node in code.Syntax.GetDescendants <BinaryExpression>())
{
if (node.Kind == SyntaxKind.ContainsExpression ||
node.Kind == SyntaxKind.NotContainsExpression ||
node.Kind == SyntaxKind.ContainsCsExpression ||
node.Kind == SyntaxKind.NotContainsCsExpression)
{
diagnostics.Add(_diagnostic.WithLocation(node.Operator));
}
}
return(diagnostics);
}
public override void Analyze(KustoCode code, List <Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants <SummarizeOperator>())
{
var badSums = node.Aggregates.GetDescendants <FunctionCallExpression>(fc =>
fc.ReferencedSymbol == Aggregates.Sum &&
fc.ArgumentList.Expressions[0].Element is BinaryExpression b &&
(b.Kind == SyntaxKind.AddExpression ||
b.Kind == SyntaxKind.SubtractExpression)
);
foreach (var bs in badSums)
{
diagnostics.Add(GetDiagnostic(bs));
}
}
}
private static CommandBase?ParseAndCreateCommand(
string script,
bool ignoreUnknownCommands)
{
try
{
var code = KustoCode.Parse(script);
var command = CreateCommand(script, code, ignoreUnknownCommands);
return(command);
}
catch (Exception ex)
{
throw new DeltaException(
$"Issue parsing script",
script,
ex);
}
}
public override void Analyze(KustoCode code, List <Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants <FunctionCallExpression>())
{
if ((node.ReferencedSymbol == Functions.IsNull ||
node.ReferencedSymbol == Functions.IsNotNull) &&
node.ArgumentList.Expressions.Count > 0 &&
node.ArgumentList.Expressions[0].Element.ResultType == ScalarTypes.String)
{
if (node.ReferencedSymbol == Functions.IsNull)
{
diagnostics.Add(_diagnostic_equals.WithLocation(node));
}
else
{
diagnostics.Add(_diagnostic_not_equals.WithLocation(node));
}
}
}
}
public override void Analyze(KustoCode code, List <Diagnostic> diagnostics, CancellationToken cancellationToken)
{
foreach (var node in code.Syntax.GetDescendants <FunctionCallExpression>())
{
if ((node.ReferencedSymbol == Functions.ToBool || node.ReferencedSymbol == Functions.ToBoolean) && node.ArgumentList.Expressions.Count > 0)
{
var firstArgumentType = node.ArgumentList.Expressions[0].Element.ResultType;
if (firstArgumentType == ScalarTypes.DateTime ||
firstArgumentType == ScalarTypes.Int ||
firstArgumentType == ScalarTypes.Decimal ||
firstArgumentType == ScalarTypes.Guid ||
firstArgumentType == ScalarTypes.Long ||
firstArgumentType == ScalarTypes.Real ||
firstArgumentType == ScalarTypes.TimeSpan)
{
diagnostics.Add(_diagnostic.WithLocation(node));
}
}
}
}
public EvaluateOperator(string args)
{
args = args.Replace("evaluate ", string.Empty);
KustoCode query;
lock (parserLock)
{
query = KustoCode.Parse(args);
}
var diagnostics = query.GetSyntaxDiagnostics()
.Select(d => $"({d.Start}..{d.Start + d.Length}): {d.Message}");
if (diagnostics.Any())
{
var errors = string.Join("\n", diagnostics);
throw new QueryParsingException($"Error parsing expression {args}: {errors}");
}
var syntax = query.Syntax.GetDescendants <Statement>()[0];
Expression = syntax.Visit(new ScalarValueConverter()) as ScalarFunction;
}
public KustoRelatedElementFinder(KustoCode code)
{
_code = code;
}
/// <summary> /// Determines if the parsed syntax can be analyzed /// </summary> private static bool CanBeAnalyzed(KustoCode code) => code.MaxDepth <= KustoCode.MaxAnalyzableSyntaxDepth;
public KustoQuickInfoBuilder(KustoCode code)
{
_code = code;
}
public KustoCodeService(KustoCode code)
: this(code.Text, code.Globals, code)
{
}
