private async Task <long> GetCountQuery(Guid dashboardId, int groupIndex, int itemIndex, ItemDuration duration, DateTime durationFrom, DateTime durationTo, string[] queryParts)
{
var item = _config.Dashboards[dashboardId].Select(d => d.Value).ToList()[groupIndex][itemIndex];
string itemQuery;
if (duration == ItemDuration.Custom)
{
itemQuery = GetQueryString(item, durationFrom, durationTo, durationFrom.GetIntervalString(durationTo));
}
else
{
var durationString = duration.GetString();
var intervalString = duration.GetIntervalString();
itemQuery = GetQueryString(item, durationString, intervalString);
}
var structuredQuery = QueryBuilder.Parse(itemQuery);
var queryGroup = new QueryGroup(structuredQuery);
queryGroup.AddParts(queryParts);
queryGroup.RemoveProjectAndSummarize();
queryGroup.Append(QueryBuilder.Parse("| summarize sum(itemCount)"));
var query = queryGroup.ToString();
var table = await AppInsightsClient.GetTableQuery(item.ApiToken.Id, item.ApiToken.Key, query);
return(table.Rows.First()[0] as long? ?? 0);
}
protected void Application_Error(Object sender, EventArgs e)
{
// Process exception
var raisedException = Server.GetLastError();
if (raisedException == null)
{
return;
}
//Add to the log
ErrorLog.WriteLine(raisedException.ToString());
//Track the exception with Application Insights if it is available
AppInsightsClient?.TrackException(raisedException);
if (!HttpContext.Current.IsCustomErrorEnabled)
{
return;
}
if (raisedException is HttpException)
{
HttpContext.Current.Response.Redirect("~/Error?code=" + ((HttpException)raisedException).GetHttpCode());
}
else
{
HttpContext.Current.Response.Redirect("~/Error");
}
}
private void ServiceFabricTraceEvent(TraceEvent data)
{
var eventId = (ushort)data.ID;
var eventIsRelevant =
(
(data.Level <= TraceEventLevel.Error && data.Level >= TraceEventLevel.Critical) ||
(
ServiceFabricEtw.Operational.ClusterEventsFilter(eventId, data) ||
ServiceFabricEtw.Operational.NodeEventsFilter(eventId, data)
));
if (!eventIsRelevant)
{
return;
}
var properties = data.PayloadNames
.Select((name, index) =>
{
if (data.PayloadValue(index) is DateTime dateTimeValue)
{
return(new KeyValuePair <string, string>(name, dateTimeValue.ToString("o")));
}
return(new KeyValuePair <string, string>(name, data.PayloadString(index, Culture)));
})
.ToDictionary(x => x.Key, x => x.Value);
properties.Add("eventID", data.ID.ToString());
properties.Add("name", data.EventName);
properties.Add("timeStamp", data.TimeStamp.ToString("o"));
AppInsightsClient.TrackTrace(data.FormattedMessage, data.ToSeverityLevel(), properties);
}
public void Stop()
{
EtwSession?.Dispose();
EtwSession = null;
AppInsightsClient?.Flush();
AppInsightsConfig = null;
}
public async Task <IItemStatus> GetStatus()
{
var requestsCount = AppInsightsClient.GetRequestsCount(ApplicationId, ApiKey, AppInsightsTimeSpan.PT1H);
var requestsFailed = AppInsightsClient.GetExceptionsServer(ApplicationId, ApiKey, AppInsightsTimeSpan.PT1H);
var requestsDuration = AppInsightsClient.GetRequestsDurationPercentile(ApplicationId, ApiKey, 90);
var requestsCount10Min = AppInsightsClient.GetRequestsCount(ApplicationId, ApiKey, AppInsightsTimeSpan.PT10M);
var requestsFailed10Min = AppInsightsClient.GetExceptionsServer(ApplicationId, ApiKey, AppInsightsTimeSpan.PT10M);
var availabilityPercentage10Min = AppInsightsClient.GetAvailabilityPercentage(ApplicationId, ApiKey, AppInsightsTimeSpan.PT10M);
await Task.WhenAll(requestsCount, requestsFailed, requestsDuration, requestsCount10Min, requestsFailed10Min, availabilityPercentage10Min);
var requests = requestsCount.Result / 60;
var responseTime = requestsDuration.Result ?? 0;
var errorRate = requestsCount.Result.HasValue && requestsFailed.Result.HasValue && requestsCount.Result.Value > 0 && requestsFailed.Result.Value >= SiteSettings.ErrorCountMinimum ? Math.Round(100.0 / requestsCount.Result.Value * requestsFailed.Result.Value, 1) : 0;
var errorRate10Min = requestsCount10Min.Result.HasValue && requestsFailed10Min.Result.HasValue && requestsCount10Min.Result > 0 && requestsFailed10Min.Result >= SiteSettings.ErrorCountMinimum ? Math.Round(100.0 / requestsCount10Min.Result.Value * requestsFailed10Min.Result.Value, 1) : 0;
var availabilityErrorLevel = availabilityPercentage10Min.Result < 50 ? ErrorLevel.Error : ErrorLevel.Normal;
if (requestsFailed.Result.HasValue && requestsFailed.Result.Value > 0 && (!requestsCount.Result.HasValue || requestsCount.Result.Value == 0))
{
errorRate = 100;
}
if (requestsFailed10Min.Result.HasValue && requestsFailed10Min.Result.Value > 0 && (!requestsCount10Min.Result.HasValue || requestsCount10Min.Result.Value == 0))
{
errorRate10Min = 100;
}
var result = new SiteStatus
{
RequestsPerMinute = requests ?? 0,
AvgResponseTime = Convert.ToInt32(responseTime),
ErrorRate = errorRate,
ErrorRate10Min = errorRate10Min,
AvgResponseTimeErrorLevel = responseTime > SiteSettings.AvgResponseTimeWarning ? ErrorLevel.Warning : ErrorLevel.Normal,
ErrorRateLevel = GetErrorRateLevel(errorRate, SiteSettings),
ErrorRateLevel10Min = GetErrorRateLevel(errorRate10Min, SiteSettings)
};
var errorLevels = new[]
{
result.AvgResponseTimeErrorLevel,
result.ErrorRateLevel,
result.ErrorRateLevel10Min,
availabilityErrorLevel
};
if (errorLevels.Any(r => r == ErrorLevel.Error))
{
result.ErrorLevel = ErrorLevel.Error;
}
else if (errorLevels.Any(r => r == ErrorLevel.Warning))
{
result.ErrorLevel = ErrorLevel.Warning;
}
else if (errorLevels.Any(r => r == ErrorLevel.Normal))
{
result.ErrorLevel = ErrorLevel.Normal;
}
return(result);
}
public ApplicationInsightsPanelViewModel(ConnectedAccount account)
{
Account = account;
_client = new AppInsightsClient(
account.StorageConnectionString.Get("appId"),
account.StorageConnectionString.Get("apiKey"));
}
public static async Task <IAnalyzerResult> Analyze(Guid appid, string apikey, StructuredQuery query, string[] queryParts)
{
query = QueryBuilder.RemoveProject(query);
query = QueryBuilder.RemoveSummarize(query);
var isExceptionsQuery = query.FirstOrDefault()?.Trim().StartsWith("exceptions", StringComparison.OrdinalIgnoreCase) == true;
QueryGroup queryGroup;
var whereQuery = @"
| where details[0].parsedStack[0].fileName != ''
| project itemCount, filename = extract('([^\\\\/]+)$', 1, tostring(details[0].parsedStack[0].fileName)), type, line = tostring(details[0].parsedStack[0].line)
| summarize sum(itemCount) by filename, type, line
| order by sum_itemCount";
if (isExceptionsQuery)
{
queryGroup = new QueryGroup(query);
queryGroup.AddParts(queryParts);
queryGroup.Append(QueryBuilder.Parse(whereQuery));
}
else
{
var exceptionsQuery = QueryBuilder.Parse(
$@"
exceptions
| {query.First(q => q.Contains("timestamp >"))}
{whereQuery}");
queryGroup = new QueryGroup(exceptionsQuery);
queryGroup.Replace(query.First().Trim(), query);
queryGroup.AddParts(queryParts);
}
var queryString = queryGroup.ToString();
var result = await AppInsightsClient.GetTableQuery(appid, apikey, queryString);
result.Columns[0].Name = "File";
result.Columns[1].Name = "Exception";
result.Columns[3].Name = "Count";
result.Columns.RemoveAt(2);
foreach (var row in result.Rows)
{
row.Add($"where details[0].parsedStack[0].fileName endswith '{row[0]}' and details[0].parsedStack[0].line == {row[2]}");
row.Add($"where details[0].parsedStack[0].fileName !endswith '{row[0]}' or details[0].parsedStack[0].line != {row[2]}");
row[0] = $"{row[0]} ({row[2]})";
var exception = (string)row[1];
row[1] = exception.Substring(exception.LastIndexOf('.') + 1);
row.RemoveAt(2);
}
return(new TableAnalyzerResult("Stacktraces", true, result));
}
public static async Task <IAnalyzerResult> Analyze(ApiToken apiToken, StructuredQuery query, string[] queryParts)
{
query = QueryBuilder.RemoveProject(query);
query = QueryBuilder.RemoveSummarize(query);
var isRequestsQuery = query.FirstOrDefault()?.Trim().StartsWith("requests", StringComparison.OrdinalIgnoreCase) == true;
QueryGroup queryGroup;
var whereQuery = @"
| project domain = extract('^(https|http)://(.*?)/', 2, url), duration, itemCount, success
| summarize duration = avg(duration), failedCount=sumif(itemCount, success == false), totalCount=sum(itemCount) by domain
| order by totalCount desc
| take 20
| project domain, totalCount, duration, failedCount, failedPercentage = 100.0 / totalCount * failedCount";
if (isRequestsQuery)
{
queryGroup = new QueryGroup(query);
queryGroup.AddParts(queryParts);
queryGroup.Append(QueryBuilder.Parse(whereQuery));
}
else
{
var requestsQuery = QueryBuilder.Parse(
$@"
requests
| {query.First(q => q.Contains("timestamp >"))}
{whereQuery}");
queryGroup = new QueryGroup(requestsQuery);
queryGroup.Replace(query.First().Trim(), query);
queryGroup.AddParts(queryParts);
}
var queryString = queryGroup.ToString();
var result = await AppInsightsClient.GetTableQuery(apiToken, queryString);
result.Columns[0].Name = "Domain";
result.Columns[1].Name = "Count";
result.Columns[2].Name = "Duration";
result.Columns[3].Name = "Failures";
result.Columns[4].Name = "Percentage";
foreach (var row in result.Rows)
{
row[2] = $"{row[2]:0} ms";
row[4] = $"{row[4]:0} %";
row.Add($"where url contains '{row[0]}'");
row.Add($"where url !contains '{row[0]}'");
}
return(new TableAnalyzerResult("Domains", true, result));
}
public static async Task <IAnalyzerResult> Analyze(ApiToken apiToken, StructuredQuery query, string[] queryParts)
{
query = QueryBuilder.RemoveProject(query);
query = QueryBuilder.RemoveSummarize(query);
var isRequestsQuery = query.FirstOrDefault()?.Trim().StartsWith("requests", StringComparison.OrdinalIgnoreCase) == true;
QueryGroup queryGroup;
var whereQuery = @"
| summarize _count=sum(itemCount) by resultCode
| project resultCode, _count
| sort by _count desc";
if (isRequestsQuery)
{
queryGroup = new QueryGroup(query);
queryGroup.AddParts(queryParts);
queryGroup.Append(QueryBuilder.Parse(whereQuery));
}
else
{
var requestsQuery = QueryBuilder.Parse(
$@"
requests
| {query.First(q => q.Contains("timestamp >"))}
{whereQuery}");
queryGroup = new QueryGroup(requestsQuery);
queryGroup.Replace(query.First().Trim(), query);
queryGroup.AddParts(queryParts);
}
var queryString = queryGroup.ToString();
var result = await AppInsightsClient.GetTableQuery(apiToken, queryString);
result.Columns[0].Name = "Status code";
result.Columns[1].Name = "Count";
foreach (var row in result.Rows)
{
row.Add($"where resultCode == {row[0]}");
row.Add($"where resultCode != {row[0]}");
}
return(new TableAnalyzerResult("Status codes", true, result));
}
public static async Task <IAnalyzerResult> Analyze(Guid appid, string apikey, StructuredQuery query, string[] queryParts)
{
query = QueryBuilder.RemoveProject(query);
query = QueryBuilder.RemoveSummarize(query);
var isExceptionsQuery = query.FirstOrDefault()?.Trim().StartsWith("exceptions", StringComparison.OrdinalIgnoreCase) == true;
QueryGroup queryGroup;
var whereQuery = @"
| summarize _count = sum(itemCount) by type
| sort by _count desc
| take 20";
if (isExceptionsQuery)
{
queryGroup = new QueryGroup(query);
queryGroup.AddParts(queryParts);
queryGroup.Append(QueryBuilder.Parse(whereQuery));
}
else
{
var exceptionsQuery = QueryBuilder.Parse(
$@"
exceptions
| {query.First(q => q.Contains("timestamp >"))}
{whereQuery}");
queryGroup = new QueryGroup(exceptionsQuery);
queryGroup.Replace(query.First().Trim(), query);
queryGroup.AddParts(queryParts);
}
var queryString = queryGroup.ToString();
var result = await AppInsightsClient.GetTableQuery(appid, apikey, queryString);
result.Columns[0].Name = "Exception type";
result.Columns[1].Name = "Count";
foreach (var row in result.Rows)
{
row.Add($"where type == '{row[0]}'");
row.Add($"where type != '{row[0]}'");
}
return(new TableAnalyzerResult("Exceptions", true, result));
}
private async Task <double> GetValueQuery(ConfigLogic.Dashboard.DashboardItem item, string query)
{
var table = await AppInsightsClient.GetTableQuery(item.ApiToken.Id, item.ApiToken.Key, query);
var value = 0.0;
if (table.Rows.Count > 0 && table.Rows[0].Count > 1 && (table.Rows[0][1] is double || table.Rows[0][1] is long))
{
value = table.Rows[0][1];
}
if (item.Total == ItemTotal.Rpm)
{
return(value / item.Duration.GetString().GetTimeSpan().TotalMinutes);
}
return(value);
}
public async Task <dynamic> Overview(Guid dashboardId, int groupIndex, int itemIndex)
{
var item = _config.Dashboards[dashboardId].Select(d => d.Value).ToList()[groupIndex][itemIndex];
var itemQuery = GetQueryString(item, item.Duration.GetString(), item.Duration.GetIntervalString());
var table = await AppInsightsClient.GetTableQuery(item.ApiToken.Id, item.ApiToken.Key, itemQuery);
var values = table.Rows.Select(row => new RowItem(row[0], row[1])).ToList();
var value = await GetValueQuery(item, GetQueryString(item, item.Duration.GetString(), "90d"));
var chart = TransformQuery(values, item.Duration.GetString().GetTimeSpan(), item.Duration.GetIntervalString().GetTimeSpan());
var max = Math.Max(item.MinChartValue, chart.Any() ? chart.Max() : 0);
var status = item.GetStatus(item, value, chart);
return(new
{
Value = item.FormatValue(value),
ChartValues = chart,
ChartMax = max,
Status = status
});
}
private async Task <List <RowItem> > GetChartValues(ConfigLogic.Dashboard.DashboardItem item, string query)
{
var table = await AppInsightsClient.GetTableQuery(item.ApiToken.Id, item.ApiToken.Key, query);
return(table.Rows.Select(row => new RowItem(row[0], row[1])).ToList());
}
public static async Task <IAnalyzerResult> Analyze(Guid appid, string apikey, StructuredQuery query, string[] queryParts)
{
query = QueryBuilder.RemoveProject(query);
query = QueryBuilder.RemoveSummarize(query);
var isExceptionsQuery = query.FirstOrDefault()?.Trim().StartsWith("exceptions", StringComparison.OrdinalIgnoreCase) == true;
QueryGroup queryGroup;
var whereQuery = @"
| project itemCount, message = strcat(outerMessage, ' \n\n ', customDimensions['AdditionalErrorDetails'], ' \n\n ', customDimensions['additionalDetails'])
| project itemCount, message = replace(@'[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}', '[GUID]', message)
| project itemCount, message = replace(@'(^|[ ''])(http:|https:)*[/]{1,2}(.*?)([ ]|$|\n)', ' [URL] ', message)
| project itemCount, message = replace(@'[0-9]', '[X]', message)
| where message !contains 'username' or message !contains 'password'
| summarize sum(itemCount) by message
| order by sum_itemCount desc
| take 20";
if (isExceptionsQuery)
{
queryGroup = new QueryGroup(query);
queryGroup.AddParts(queryParts);
queryGroup.Append(QueryBuilder.Parse(whereQuery));
}
else
{
var exceptionsQuery = QueryBuilder.Parse(
$@"
exceptions
| {query.First(q => q.Contains("timestamp >"))}
{whereQuery}");
queryGroup = new QueryGroup(exceptionsQuery);
queryGroup.Replace(query.First().Trim(), query);
queryGroup.AddParts(queryParts);
}
var queryString = queryGroup.ToString();
var result = await AppInsightsClient.GetTableQuery(appid, apikey, queryString);
result.Columns[0].Name = "Exception message";
result.Columns[1].Name = "Count";
foreach (var row in result.Rows)
{
var rowText = (string)row[0];
var commandText = ((string)row[0]).Trim();
rowText = rowText.Replace("[X]", "X");
rowText = Regex.Replace(rowText, "[X]+", "X");
var lines = new List <string>();
foreach (var line in Regex.Split(rowText, @"\n").SelectMany(line => Regex.Split(Regex.Replace(line, @"\. ", ".. "), @"\. ")))
{
var trimmedLine = Regex.Replace(line, @"^[ ]*\[[A-Z]+\][ ]*", "").Trim();
if (!string.IsNullOrWhiteSpace(trimmedLine) && lines.All(l => l != trimmedLine))
{
lines.Add(trimmedLine);
}
}
rowText = string.Join("\n", lines);
row[0] = rowText.Trim();
var matches = Regex.Matches(commandText, @"(^|\])(.*?)($|\[)");
if (matches.Any(match => match.Groups[2].Length > 10))
{
commandText = matches
.First(match => match.Groups[2].Length > 10)
.Groups[2].Value.Trim();
}
commandText = commandText.Replace("'", @"\'");
row.Add($"where outerMessage contains '{commandText}' or customDimensions['AdditionalErrorDetails'] contains '{commandText}' or customDimensions['additionalDetails'] contains '{commandText}'");
row.Add($"where outerMessage !contains '{commandText}' and customDimensions['AdditionalErrorDetails'] !contains '{commandText}' and customDimensions['additionalDetails'] !contains '{commandText}'");
}
return(new TableAnalyzerResult("Exception messages", true, result));
}
public Task <double?> GetStatus()
{
return(AppInsightsClient.TryGetTelemetryQuery(ApiToken.ApplicationId, ApiToken.ApiKey, Query));
}
/// <summary>
/// Every so often clean up the files.
/// </summary>
public static void CleanupFiles(string directoryPath, bool forceNow = false)
{
if (forceNow)
{
NextCleanup = DateTime.Now;
}
if (ShouldCleanup() && directoryPath != null)
{
try
{
var directory = $"{Path.GetDirectoryName(directoryPath)}{Path.DirectorySeparatorChar}";
var postersDeleted = 0;
var sharedDeleted = 0;
var tempPostersDeleted = 0;
var twitterDeleted = 0;
// Loop through the MoviePosters.
foreach (var file in Directory.GetFiles(directory, $"{MOVIE_POSTER_PREFIX}*"))
{
if (file.IndexOf(".temp") > 0)
{
// Just delete the temp files each possible pass.
File.Delete(file);
tempPostersDeleted++;
}
else if (File.GetCreationTime(file) < DateTime.Now.AddDays(MOVIE_EXPIRATION_DAYS * -1))
{
File.Delete(file);
postersDeleted++;
}
}
// Loop through the Shared images.
foreach (var file in Directory.GetFiles(directory, "Shared_*"))
{
if (File.GetCreationTime(file) < DateTime.Now.AddMinutes(SHARED_EXPIRATION_MINUTES * -1))
{
File.Delete(file);
sharedDeleted++;
}
}
// Loop through the Twitter images.
foreach (var file in Directory.GetFiles(directory, "Twitter_*"))
{
if (File.GetCreationTime(file) < DateTime.Now.AddMinutes(TWITTER_EXPIRATION_MINUTES * -1))
{
File.Delete(file);
twitterDeleted++;
}
}
NextCleanup = DateTime.Now.AddMinutes(SHARED_EXPIRATION_MINUTES);
AppInsightsClient?.TrackTrace("Cleanup Files: ", SeverityLevel.Information
, new Dictionary <string, string> {
{ "tempPostersDeleted", tempPostersDeleted.ToString() }
, { "postersDeleted", postersDeleted.ToString() }
, { "sharedDeleted", sharedDeleted.ToString() }
, { "twitterDeleted", twitterDeleted.ToString() }
, { "NextCleanup", NextCleanup.ToString() }
});
}
finally
{
// Make sure this gets set back NO MATTER WHAT, otherwise it will NEVER clean up!!
_isCleaningUp = false;
}
}
}
