private void CreateEmptyPackageReport(string packageId)
{
Log.Info($"CreateEmptyPackageReport for {packageId}");
// All blob names use lower case identifiers in the NuGet Gallery Azure Blob Storage
string name = PackageReportDetailBaseName + packageId.ToLowerInvariant();
CreateBlob(name + ".json", JsonContentType, ReportHelpers.ToStream(new JObject()));
}
// for the initial release we will run New and Old reports in parallel
// (the difference is that new reports contain more details)
// then when we are happy with our new deployment we will drop the old
private void CreatePackageReport(string packageId)
{
Log.Info(string.Format("CreatePackageReport for {0}", packageId));
// All blob names use lower case identifiers in the NuGet Gallery Azure Blob Storage
string name = PackageReportDetailBaseName + packageId.ToLowerInvariant();
JObject report = CreateJsonContent(packageId);
CreateBlob(name + ".json", JsonContentType, ReportHelpers.ToStream(report));
}
public override void ExecuteCommand()
{
JavaScriptSerializer js = new JavaScriptSerializer();
AlertThresholds thresholdValues = js.Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
int error = thresholdValues.DatabaseSizePercentErrorThreshold;
int warning = thresholdValues.DatabaseSizePercentWarningThreshold;
List <DatabaseSize> dbSizeDetails = new List <DatabaseSize>();
// dbSizeDetails.Add(GetDataSize(PrimaryConnectionString.ConnectionString,threshold));
dbSizeDetails.Add(GetDataSize(LegacyConnectionString.ConnectionString, error, warning));
dbSizeDetails.Add(GetDataSize(WarehouseConnectionString.ConnectionString, error, warning));
var json = js.Serialize(dbSizeDetails);
ReportHelpers.CreateBlob(StorageAccount, "DBSize.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
private void CreateInstanceStateReport(XmlDocument doc)
{
XmlNodeList roleInstanceNodes = doc.GetElementsByTagName("RoleInstance", "http://schemas.microsoft.com/windowsazure");
int roleInstanceCount = roleInstanceNodes.Count;
List <Tuple <string, string> > instanceStatuses = new List <Tuple <string, string> >();
string[] invalidStatus = { "RestartingRole", "CyclingRole", "FailedStartingRole", "FailedStartingVM", "UnresponsiveRole", "StoppedDeallocated", "Preparing" };
int unReadyInstanceCount = 0;
List <string> unReadyInstanceStatus = new List <string>();
foreach (XmlNode node in roleInstanceNodes)
{
string instanceName = node.ChildNodes[1].InnerText;
string instanceStatus = node.ChildNodes[2].InnerText;
instanceStatuses.Add(new Tuple <string, string>(instanceName, instanceStatus));
//Loist of instance status @ http://msdn.microsoft.com/en-us/library/azure/ee460804.aspx#RoleInstanceList. Only Ready and unknown are acceptable.
if (!instanceStatus.Equals("ReadyRole", StringComparison.OrdinalIgnoreCase))
{
unReadyInstanceCount++;
unReadyInstanceStatus.Add(instanceName + "-" + instanceStatus);
}
if (invalidStatus.Contains(instanceStatus))
{
string extendedStatus = node.ChildNodes[3].InnerText;
new SendAlertMailTask
{
AlertSubject = string.Format("Cloud service {0} not in good state", ServiceName),
Details = string.Format("The status of the instance {0} in cloud service {1} is {2}. Addtional information on the instance statuses : {3}", instanceName, ServiceName, instanceStatus, extendedStatus),
AlertName = string.Format("Cloud service {0} not in good state", ServiceName), //ensure uniqueness in Alert name as that is being used incident key in pagerduty.
Component = "CloudService",
Level = "Error"
}.ExecuteCommand();
}
}
if ((roleInstanceCount - unReadyInstanceCount) < 2)
{
string unReadyReport = string.Join(",", unReadyInstanceStatus.ToArray());
new SendAlertMailTask
{
AlertSubject = string.Format("Multiple instances in Cloud service {0} not in good state", ServiceName),
Details = string.Format("Multiple instances in service '{0}' not in good state. The list of instance names and statuses are as follows : {1}", ServiceName, unReadyReport),
AlertName = string.Format("Multiple instances in Cloud service {0} not in good state", ServiceName), //ensure uniqueness in Alert name as that is being used incident key in pagerduty.
Component = "CloudService " + ServiceName,
Level = "Error"
}.ExecuteCommand();
}
JArray reportObject = ReportHelpers.GetJson(instanceStatuses);
ReportHelpers.CreateBlob(StorageAccount, ServiceName + "InstanceStatus.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
public override void ExecuteCommand()
{
AlertThresholds thresholds = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
List <ElmahError> listOfErrors = new List <ElmahError>();
RefreshElmahError RefreshExecute = new RefreshElmahError(StorageAccount, ContainerName, LastNHours, ElmahAccountCredentials);
listOfErrors = RefreshExecute.ExecuteRefresh();
foreach (ElmahError error in listOfErrors)
{
if (error.Severity == 0)
{
if (error.Occurecnes > thresholds.ElmahCriticalErrorPerHourAlertErrorThreshold && LastNHours == 1)
{
new SendAlertMailTask
{
AlertSubject = string.Format("Error: Elmah Error Alert activated for {0}", error.Error),
Details = String.Format("Number of {0} exceeded Error threshold limit during the last hour.Threshold error count per hour : {1}, Events recorded in the last hour: {2}", error.Error, thresholds.ElmahCriticalErrorPerHourAlertErrorThreshold, error.Occurecnes.ToString()),
AlertName = string.Format("Error: Elmah Error Alert for {0}", error.Error),
Component = "Web Server",
Level = "Error"
}.ExecuteCommand();
}
else if (error.Occurecnes > thresholds.ElmahCriticalErrorPerHourAlertWarningThreshold && LastNHours == 1)
{
new SendAlertMailTask
{
AlertSubject = string.Format("Warning: Elmah Error Alert activated for {0}", error.Error),
Details = String.Format("Number of {0} exceeded Warning threshold limit during the last hour.Threshold error count per hour : {1}, Events recorded in the last hour: {2}", error.Error, thresholds.ElmahCriticalErrorPerHourAlertWarningThreshold, error.Occurecnes.ToString()),
AlertName = string.Format("Warning: Elmah Error Alert for {0}", error.Error),
Component = "Web Server",
Level = "Warning"
}.ExecuteCommand();
}
}
}
var json = new JavaScriptSerializer().Serialize(listOfErrors);
ReportHelpers.CreateBlob(StorageAccount, "ElmahErrorsDetailed" + LastNHours.ToString() + "hours.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
public override void ExecuteCommand()
{
sqlQueryForDbAge = string.Format("select create_date from sys.databases where name = '{0}'", DbName);
thresholdValues = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
List <Tuple <string, string> > jobOutputs = new List <Tuple <string, string> >();
jobOutputs.Add(new Tuple <string, string>("SyncPackagesToFailoverDC", CheckLagBetweenDBAndBlob()));
jobOutputs.Add(new Tuple <string, string>("ImportCompletionStatus", CheckForInCompleteDBImport()));
JArray reportObject = ReportHelpers.GetJson(jobOutputs);
ReportHelpers.CreateBlob(StorageAccount, "RunBackgroundCheckForFailoverDCReport.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
private void CreateDepolymentIdReport(XmlDocument doc)
{
XmlNodeList depolymentIdNode = doc.GetElementsByTagName("PrivateID", "http://schemas.microsoft.com/windowsazure");
string depolyId = depolymentIdNode[0].InnerText;
var json = new JavaScriptSerializer().Serialize(depolyId);
ReportHelpers.CreateBlob(StorageAccount, "DeploymentId_" + ServiceName + ".json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
private void CreatereportForSupportRequestSummary()
{
List <Tuple <string, string> > summary = new List <Tuple <string, string> >();
summary.Add(new Tuple <string, string>("No. of open requests ", GetWorkItemsForLastNDays("30").ToString()));
summary.Add(new Tuple <string, string>("No. of requests in last 7 days", GetWorkItemsForLastNDays("7").ToString()));
summary.Add(new Tuple <string, string>("Average reslution time for last 7 days", GetResolutionRateForLastNDays("7").ToString()));
summary.Add(new Tuple <string, string>("No. of requests in last 30 days", GetWorkItemsForLastNDays("30").ToString()));
summary.Add(new Tuple <string, string>("Average reslution time for last 30 days", GetResolutionRateForLastNDays("30").ToString()));
JArray reportObject = ReportHelpers.GetJson(summary);
ReportHelpers.CreateBlob(StorageAccount, "SupportRequestSummaryReport.json", "dashboard", "application/json", ReportHelpers.ToStream(reportObject));
}
public override void ExecuteCommand()
{
StopWatches timer = new StopWatches();
DateTime today = DateTime.Today;
string day = string.Format("{0:yyyy-MM-dd}", today);
string version = string.Empty;
string file = Path.Combine(Environment.CurrentDirectory, TestPackageName + ".nupkg");
string newPackage = GetNewPackage(file, out version);
//upload
Console.WriteLine("Pushing :{0}", newPackage);
long UploadTimeElapsed = UploadPackage(timer, newPackage, Source, ApiKey);
ReportHelpers.AppendDatatoBlob(StorageAccount, ("UploadPackageTimeElapsed" + day + ".json"), new Tuple <string, string>(string.Format("{0:HH:mm}", DateTime.Now), UploadTimeElapsed.ToString()), 48, ContainerName);
File.Delete("backup");
//download
long DownloadTimeElapsed = -1;
Task <string> result = null;
result = DownloadPackageFromFeed(timer, TestPackageName, version, out DownloadTimeElapsed);
Console.WriteLine(result.Status);
DownloadTimeElapsed = timer.DownloadTimeElapsed.ElapsedMilliseconds;
ReportHelpers.AppendDatatoBlob(StorageAccount, ("DownloadPackageTimeElapsed" + day + ".json"), new Tuple <string, string>(string.Format("{0:HH:mm}", DateTime.Now), DownloadTimeElapsed.ToString()), 48, ContainerName);
//search
long SearchTimeElapsed = -1;
//SearchPackage is called until the uploaded package is seen in the search result
while (SearchTimeElapsed == -1)
{
SearchTimeElapsed = SearchPackage(timer, TestPackageName, version);
}
ReportHelpers.AppendDatatoBlob(StorageAccount, ("SearchPackageTimeElapsed" + day + ".json"), new Tuple <string, string>(string.Format("{0:HH:mm}", DateTime.Now), SearchTimeElapsed.ToString()), 48, ContainerName);
//catalog lag
JToken timeStampCatalog;
int CatalogLag = DBToCatalogLag(timer, TestPackageName, out timeStampCatalog);
ReportHelpers.AppendDatatoBlob(StorageAccount, ("CatalogLag" + day + ".json"), new Tuple <string, string>(string.Format("{0:HH:mm}", DateTime.Now), CatalogLag.ToString()), 48, ContainerName);
ReportHelpers.CreateBlob(StorageAccount, ("LastCatalogTimeStamp.json"), ContainerName, "SqlDateTime", ReportHelpers.ToStream(timeStampCatalog));
//resolver lag
JToken timeStampResolver;
double ResolverLag = CatalogToResolverLag(out timeStampResolver);
ReportHelpers.AppendDatatoBlob(StorageAccount, ("ResolverLag" + day + ".json"), new Tuple <string, string>(string.Format("{0:HH:mm}", DateTime.Now), ResolverLag.ToString()), 48, ContainerName);
ReportHelpers.CreateBlob(StorageAccount, ("LastResolverTimeStamp.json"), ContainerName, "SqlDateTime", ReportHelpers.ToStream(timeStampResolver));
SendAlerts(UploadTimeElapsed, DownloadTimeElapsed, SearchTimeElapsed, CatalogLag, ResolverLag);
}
public override void ExecuteCommand()
{
NetworkCredential nc = new NetworkCredential(UserName, Password);
WebRequest request = WebRequest.Create("https://api.pingdom.com/api/2.0/checks");
request.Credentials = nc;
request.Headers.Add(AppKey);
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
using (var reader = new StreamReader(respose.GetResponseStream()))
{
JavaScriptSerializer js = new JavaScriptSerializer();
var objects = js.Deserialize <dynamic>(reader.ReadToEnd());
foreach (var o in objects["checks"])
{
List <Tuple <string, string> > summary = GetCheckSummaryAvgForSpecifiedTimeSpan(o["id"]);
JArray reportObject = ReportHelpers.GetJson(summary);
string checkAlias = o["name"].ToString();
checkAlias = checkAlias.Replace(" ", ".");
checkAlias = checkAlias.Replace("(", "").Replace(")", "");
ReportHelpers.CreateBlob(StorageAccount, checkAlias + Frequency + "Report.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
}
}
public override void ExecuteCommand()
{
X509Certificate cert = X509Certificate.CreateFromCertFile(CertificateName);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(string.Format("https://management.core.windows.net/{0}/services/WATM/profiles/{1}/definitions", SubscriptionId, ProfileName));
request.ClientCertificates.Add(cert);
request.Headers.Add("x-ms-version: 2014-02-01");
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
//Schema of the response would be as specified in http://msdn.microsoft.com/en-us/library/azure/hh758251.aspx
Console.WriteLine(respose);
using (var reader = new StreamReader(respose.GetResponseStream()))
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(reader.ReadToEnd());
XmlNodeList parentNode = doc.GetElementsByTagName("Endpoint", "http://schemas.microsoft.com/windowsazure");
List <Tuple <string, string> > endpointValues = new List <Tuple <string, string> >();
foreach (XmlNode node in parentNode)
{
string endpointName = node.ChildNodes[0].InnerText;
string endpointStatus = node.ChildNodes[3].InnerText;
Console.WriteLine(string.Format("End point name {0}, status {1}", endpointName, endpointStatus));
endpointValues.Add(new Tuple <string, string>(endpointName, endpointStatus));
if (!endpointStatus.Equals("Online", StringComparison.OrdinalIgnoreCase))
{
new SendAlertMailTask
{
AlertSubject = string.Format("Error: Traffic manager endpoint alert activated for {0}", endpointName),
Details = string.Format("The status of the endpoint {0} monitoring by traffic manager {1} is {2}", endpointName, ProfileName, endpointStatus),
AlertName = "Error: Alert for TrafficManagerEndpoint",
Component = "TrafficManager",
Level = "Error"
}.ExecuteCommand();
}
}
JArray reportObject = ReportHelpers.GetJson(endpointValues);
ReportHelpers.CreateBlob(StorageAccount, "TrafficManagerStatus.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
}
private void CreateReportForOperationTask()
{
DateTime date = DateTime.UtcNow.AddDays(-LastNDays);
string[] agentVersion = new JavaScriptSerializer().Deserialize <string[]>(ReportHelpers.Load(StorageAccount, "agentVersion.json", ContainerName));
string[] Operation = new JavaScriptSerializer().Deserialize <string[]>(ReportHelpers.Load(StorageAccount, "OperationType.json", ContainerName));
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
foreach (string opt in Operation)
{
List <agentRequest> result = new List <agentRequest>();
foreach (string version in agentVersion)
{
string major = version[0].ToString();
string minor = version[2].ToString();
try
{
var requests = dbExecutor.Query <Int32>(string.Format(sqlQueryForOperation, major, minor, opt, date.ToString("yyyy-MM-dd"))).SingleOrDefault();
result.Add(new agentRequest(version, requests));
}
catch
{
result.Add(new agentRequest(version, 0));
}
}
var json = new JavaScriptSerializer().Serialize(result);
ReportHelpers.CreateBlob(StorageAccount, opt + LastNDays.ToString() + "Day.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
}
}
}
private void CreateRestoreReportForVSTask()
{
string[] VsQuery = new JavaScriptSerializer().Deserialize <string[]>(ReportHelpers.Load(StorageAccount, "VsVersion.json", ContainerName));
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
DateTime date = DateTime.UtcNow.AddDays(-LastNDays);
List <VsRequest> requests = new List <VsRequest>();
foreach (string each in VsQuery)
{
try
{
var request = dbExecutor.Query <Int32>(string.Format(sqlQueryForVSRestore, date.ToString("yyyy-MM-dd"), each)).SingleOrDefault();
requests.Add(new VsRequest("VS" + each, request.ToString()));
}
catch
{
requests.Add(new VsRequest("VS" + each, "0"));
}
}
var json = new JavaScriptSerializer().Serialize(requests);
ReportHelpers.CreateBlob(StorageAccount, "VsRestoreTrend" + LastNDays.ToString() + "Day.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
}
}
private void CreateReportForDBCPUUsage()
{
List <Tuple <string, string> > usageDataPoints = new List <Tuple <string, string> >();
var masterConnectionString = Util.GetMasterConnectionString(ConnectionString.ConnectionString);
var currentDbName = Util.GetDbName(ConnectionString.ConnectionString);
using (var sqlConnection = new SqlConnection(masterConnectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
List <DateTime> lastNTimeEntries = dbExecutor.Query <DateTime>(string.Format("select distinct Top(5) time from sys.resource_usage where database_name = '{0}' order by time desc", currentDbName.ToString())).ToList();
foreach (DateTime time in lastNTimeEntries)
{
Console.WriteLine("Time ..................." + time.ToString());
var usageSeconds = dbExecutor.Query <Int32>(string.Format("select Sum(usage_in_seconds) from sys.resource_usage where time = '{0}' AND database_name = '{1}'", time.ToString(), currentDbName)).SingleOrDefault();
usageDataPoints.Add(new Tuple <string, string>(String.Format("{0:HH:mm}", time.ToLocalTime()), usageSeconds.ToString()));
}
}
usageDataPoints.Reverse(); //reverse it as the array returned will have latest hour as first entry.
JArray reportObject = ReportHelpers.GetJson(usageDataPoints);
ReportHelpers.CreateBlob(StorageAccount, "DBCPUTime" + string.Format("{0:MMdd}", DateTime.Now) + ".json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
}
private void CreateReportForIndexFragmentation()
{
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
AlertThresholds thresholdValues = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
var fragmentationDetails = dbExecutor.Query <DatabaseIndex>(string.Format(sqlQueryForIndexFragmentation, thresholdValues.DatabaseIndexFragmentationPercentErrorThreshold));
var json = new JavaScriptSerializer().Serialize(fragmentationDetails);
ReportHelpers.CreateBlob(StorageAccount, "DBIndexFragmentation.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
}
}
}
private void CreateReportForDataBaseEvents()
{
var masterConnectionString = Util.GetMasterConnectionString(ConnectionString.ConnectionString);
var currentDbName = Util.GetDbName(ConnectionString.ConnectionString);
using (var sqlConnection = new SqlConnection(masterConnectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
var usageSeconds = dbExecutor.Query <DatabaseEvent>(string.Format("select start_time, end_time,event_type,event_count,description from sys.event_log where start_time>='{0}' and start_time<='{1}' and database_name = '{2}' and severity = 2", DateTime.UtcNow.AddHours(-LastNHours).ToString("yyyy-MM-dd hh:mm:ss"), DateTime.UtcNow.ToString("yyyy-MM-dd hh:mm:ss"), currentDbName));
var json = new JavaScriptSerializer().Serialize(usageSeconds);
ReportHelpers.CreateBlob(StorageAccount, "DBDetailed" + LastNHours.ToString() + "Hour.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
var throttlingEventCount = dbExecutor.Query <Int32>(string.Format("select count(*) from sys.event_log where start_time>='{0}' and start_time<='{1}' and database_name = '{2}' and (event_type Like 'throttling%' or event_type Like 'deadlock')", DateTime.UtcNow.AddHours(-1).ToString("yyyy-MM-dd hh:mm:ss"), DateTime.UtcNow.ToString("yyyy-MM-dd hh:mm:ss"), currentDbName)).SingleOrDefault();
var additional_data = dbExecutor.Query <string>(string.Format("select additional_data from sys.event_log where start_time>='{0}' and start_time<='{1}' and database_name = '{2}' and (event_type Like 'throttling%' or event_type Like 'deadlock')", DateTime.UtcNow.AddHours(-1).ToString("yyyy-MM-dd hh:mm:ss"), DateTime.UtcNow.ToString("yyyy-MM-dd hh:mm:ss"), currentDbName));
AlertThresholds thresholdValues = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
StringBuilder sb = new StringBuilder();
foreach (string data in additional_data)
{
if (data != null)
{
sb.Append(data + "\n");
}
}
if (throttlingEventCount > thresholdValues.DatabaseThrottlingEventErrorThreshold && LastNHours == 1)
{
new SendAlertMailTask
{
AlertSubject = "Error: SQL Azure DB alert activated for throttling/deadlock event",
Details = string.Format("Number of events exceeded threshold for DB throttling/deadlock events. Error Threshold count : {0}, events noticed in last hour : {1}, all additional data is {2}", thresholdValues.DatabaseThrottlingEventErrorThreshold, throttlingEventCount, sb.ToString()),
AlertName = "Error: SQL Azure DB throttling/deadlock event",
Component = "SQL Azure Database",
Level = "Error"
}.ExecuteCommand();
}
else if (throttlingEventCount > thresholdValues.DatabaseThrottlingEventWarningThreshold && LastNHours == 1)
{
new SendAlertMailTask
{
AlertSubject = "Warning: SQL Azure DB alert activated for throttling/deadlock event",
Details = string.Format("Number of events exceeded threshold for DB throttling/deadlock events. Warning Threshold count : {0}, events noticed in last hour : {1}, all additional data is {2}", thresholdValues.DatabaseThrottlingEventWarningThreshold, throttlingEventCount, sb.ToString()),
AlertName = "Warning: SQL Azure DB throttling/deadlock event",
Component = "SQL Azure Database",
Level = "Warning"
}.ExecuteCommand();
}
}
}
}
public override void ExecuteCommand()
{
int lastNhour = 24;
List <WorkInstanceDetail> jobDetail = new List <WorkInstanceDetail>();
List <WorkJobInstanceDetails> instanceDetails = getWorkjobInstance();
foreach (WorkJobInstanceDetails job in instanceDetails)
{
int invocationCount = 0;
double totalRunTime = 0;
int faultCount = 0;
int faultRate = 0;
int runtime = 0;
Dictionary <string, List <string> > ErrorList = new Dictionary <string, List <string> >();
string AdminKey = WorkServiceAdminKey;
if (job.url.Contains("api-work-1"))
{
AdminKey = WorkServiceFailoverAdminKey;
}
NetworkCredential nc = new NetworkCredential(WorkServiceUserName, AdminKey);
WebRequest request = WebRequest.Create(string.Format("{0}/instances/{1}?limit={2}", job.url, job.JobInstanceName, (lastNhour * 60) / job.FrequencyInMinutes));
request.Credentials = nc;
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
using (var reader = new StreamReader(respose.GetResponseStream()))
{
JavaScriptSerializer js = new JavaScriptSerializer();
js.MaxJsonLength = Int32.MaxValue;
var objects = js.Deserialize <List <WorkJobInvocation> >(reader.ReadToEnd());
WorkJobInvocation lastJob;
string lastCompleted = string.Empty;
if (objects.Any((item => item.status.Equals("Executed") && item.result.Equals("Completed"))))
{
lastJob = objects.Where(item => item.status.Equals("Executed") && item.result.Equals("Completed")).ToList().FirstOrDefault();
}
else
{
lastJob = objects.FirstOrDefault();
}
if (lastJob != null)
{
lastCompleted = string.Format("{0} mins ago", Convert.ToInt32(DateTime.Now.Subtract(lastJob.completedAt).TotalMinutes));
}
else
{
lastCompleted = "N/A";
}
foreach (WorkJobInvocation each in objects)
{
if (each.completedAt >= DateTime.Now.AddHours(-lastNhour))
{
invocationCount++;
totalRunTime += each.completedAt.Subtract(each.queuedAt).TotalSeconds;
if (each.result.Equals("Faulted"))
{
faultCount++;
string message = getResultMessage(each.resultMessage);
if (ErrorList.ContainsKey(message))
{
if (ErrorList[message].Count < 5)
{
ErrorList[message].Add(each.logUrl);
}
}
else
{
List <string> LogUrl = new List <string>();
LogUrl.Add(each.logUrl);
ErrorList.Add(message, LogUrl);
}
}
}
}
if (invocationCount != 0)
{
faultRate = (faultCount * 100 / invocationCount);
runtime = ((int)(totalRunTime / invocationCount));
}
jobDetail.Add(new WorkInstanceDetail(job.JobInstanceName, job.FrequencyInMinutes + "mins", lastCompleted, runtime.ToString() + "s", invocationCount.ToString(), faultCount.ToString(), faultRate, ErrorList));
}
}
List <WorkServiceAdmin> allkey = new List <WorkServiceAdmin>();
allkey.Add(new WorkServiceAdmin(WorkServiceUserName, WorkServiceAdminKey));
allkey.Add(new WorkServiceAdmin(WorkServiceUserName, WorkServiceFailoverAdminKey));
var json = new JavaScriptSerializer().Serialize(jobDetail);
var key = new JavaScriptSerializer().Serialize(allkey);
ReportHelpers.CreateBlob(StorageAccount, string.Format("WorkJobDetail{0:MMdd}.json", DateTime.Now), ContainerName, "application/json", ReportHelpers.ToStream(json));
}
private void GetDetailedReportForCheck(string checkAlias)
{
DateTime startingTime = DateTime.Now.AddHours(DateTime.Now.Hour * -1).AddDays(NoOfDays * -1); //get the midnight time for today to create separate report for each day.
for (int j = 0; j <= NoOfDays; j++)
{
int i = 0;
List <Tuple <string, string> > summaryValues = new List <Tuple <string, string> >();
while (i <= 23) //get th values for each hour. TBD : For today, there may not be values for all 24 hours and they are being filled as zero.Need to fix it.
{
long fromTime = 0;
long toTime = 0;
fromTime = UnixTimeStampUtility.GetUnixTimestampSeconds(startingTime.ToUniversalTime());
toTime = UnixTimeStampUtility.GetUnixTimestampSeconds(startingTime.AddHours(1).ToUniversalTime());
NetworkCredential nc = new NetworkCredential(UserName, Password);
WebRequest request = WebRequest.Create(string.Format("https://api.pingdom.com/api/2.0/summary.average/{0}?from={1}&to={2}", CheckId, fromTime, toTime));
request.Credentials = nc;
request.Headers.Add(AppKey);
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
using (var reader = new StreamReader(respose.GetResponseStream()))
{
JavaScriptSerializer js = new JavaScriptSerializer();
var summaryObject = js.Deserialize <dynamic>(reader.ReadToEnd());
foreach (var summary in summaryObject["summary"])
{
foreach (var status in summary.Value)
{
//Get the average response time and store it to the JSON object.
if (status.Key == "avgresponse")
{
summaryValues.Add(new Tuple <string, string>(String.Format("{0:HH:mm}", UnixTimeStampUtility.DateTimeFromUnixTimestampSeconds(fromTime).ToLocalTime()), status.Value.ToString()));
}
}
}
}
i++;
startingTime = startingTime.AddHours(1);
}
JArray reportObject = ReportHelpers.GetJson(summaryValues);
ReportHelpers.CreateBlob(StorageAccount, checkAlias + string.Format("{0:MMdd}", startingTime.AddHours(-1)) + "DetailedReport.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
}
public override void ExecuteCommand()
{
thresholdValues = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
List <Tuple <string, string> > jobOutputs = new List <Tuple <string, string> >();
jobOutputs.Add(new Tuple <string, string>("PackageStatics", CheckoutForPackageStatics()));
//jobOutputs.Add(new Tuple<string, string>("PurgePackageStatistics", CheckForPurgePackagStatisticsJob()));
jobOutputs.Add(new Tuple <string, string>("HandleQueuedPackageEdits", CheckForHandleQueuedPackageEditJob()));
// jobOutputs.Add(new Tuple<string, string>("BackupPackages", CheckForBackupPackagesJob())); commenting out this check temporarily as ListBlobs on ng-backups container is giving error.
JArray reportObject = ReportHelpers.GetJson(jobOutputs);
ReportHelpers.CreateBlob(StorageAccount, "RunBackGroundChecksForWorkerJobsReport.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
private void CreateWeeklyStatReportFor(string connectionString, string sqlQuery, string reportName)
{
startingTime = new DateTime(Year, UnixTimeStampUtility.GetMonthNumber(Month), 01); //initialize to day 01 of the given month.
DateTime monthEndTime = new DateTime(Year, UnixTimeStampUtility.GetMonthNumber(Month), UnixTimeStampUtility.GetDaysInMonth(Month));
List <Tuple <string, string> > uploadsDataPoints = new List <Tuple <string, string> >();
int week = 1;
using (var sqlConnection = new SqlConnection(connectionString))
{
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
while (startingTime <= monthEndTime)
{
DateTime endTime = startingTime.AddDays(7);
if (endTime > monthEndTime)
{
endTime = monthEndTime;
}
try
{
var count = dbExecutor.Query <Int32>(string.Format(sqlQuery, startingTime.ToString("yyyy-MM-dd"), endTime.ToString("yyyy-MM-dd"))).SingleOrDefault();
uploadsDataPoints.Add(new Tuple <string, string>("Week" + week++, count.ToString()));
}
catch (NullReferenceException)
{
uploadsDataPoints.Add(new Tuple <string, string>("Week" + week++, "0"));
}
startingTime = startingTime.AddDays(7);
}
}
}
JArray reportObject = ReportHelpers.GetJson(uploadsDataPoints);
ReportHelpers.CreateBlob(ReportStorage, reportName + Month + "MonthlyReport.json", "dashboard", "application/json", ReportHelpers.ToStream(reportObject));
}
public override void ExecuteCommand()
{
int lastNhour = 24;
List <WorkInstanceDetail> jobDetail = new List <WorkInstanceDetail>();
List <WorkJobInstanceDetails> instanceDetails = getWorkjobInstance();
foreach (WorkJobInstanceDetails job in instanceDetails)
{
int invocationCount = 0;
double totalRunTime = 0;
int faultCount = 0;
int faultRate = 0;
int runtime = 0;
Dictionary <string, List <string> > ErrorList = new Dictionary <string, List <string> >();
string AdminKey = WorkServiceAdminKey;
if (job.url.Contains("api-work-1"))
{
AdminKey = WorkServiceFailoverAdminKey;
}
NetworkCredential nc = new NetworkCredential(WorkServiceUserName, AdminKey);
//get all invocations in last 24 hours or last 10 invocations
int no = (lastNhour * 60) / job.FrequencyInMinutes;
if (no < 10)
{
no = 10;
}
WebRequest request = WebRequest.Create(string.Format("{0}/instances/{1}?limit={2}", job.url, job.JobInstanceName, no));
request.Credentials = nc;
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
using (var reader = new StreamReader(respose.GetResponseStream()))
{
JavaScriptSerializer js = new JavaScriptSerializer();
js.MaxJsonLength = Int32.MaxValue;
var objects = js.Deserialize <List <WorkJobInvocation> >(reader.ReadToEnd());
WorkJobInvocation lastJob;
bool alert = false;
string lastCompleted = string.Empty;
if (objects.Any((item => item.status.Equals("Executed") && item.result.Equals("Completed"))))
{
lastJob = objects.Where(item => item.status.Equals("Executed") && item.result.Equals("Completed")).ToList().FirstOrDefault();
}
else
{
lastJob = objects.FirstOrDefault();
}
if (lastJob != null)
{
lastCompleted = string.Format("{0} mins ago", Convert.ToInt32(DateTime.Now.Subtract(lastJob.completedAt).TotalMinutes));
}
else
{
lastCompleted = "N/A";
}
foreach (WorkJobInvocation each in objects)
{
if (each.result.Equals("Incomplete", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (each.completedAt >= DateTime.Now.AddHours(-1))
{
alert = true; // check there is any failure happened in last one hour
}
invocationCount++;
totalRunTime += each.completedAt.Subtract(each.queuedAt).TotalSeconds;
if (each.result.Equals("Faulted"))
{
faultCount++;
string message = getResultMessage(each.resultMessage);
if (ErrorList.ContainsKey(message))
{
if (ErrorList[message].Count < 5)
{
ErrorList[message].Add(each.logUrl);
}
}
else
{
List <string> LogUrl = new List <string>();
LogUrl.Add(each.logUrl);
ErrorList.Add(message, LogUrl);
}
}
}
if (invocationCount != 0)
{
faultRate = (faultCount * 100 / invocationCount);
runtime = ((int)(totalRunTime / invocationCount));
}
jobDetail.Add(new WorkInstanceDetail(job.JobInstanceName, job.FrequencyInMinutes + "mins", lastCompleted, runtime.ToString() + "s", invocationCount.ToString(), faultCount.ToString(), faultRate, ErrorList));
AlertThresholds thresholdValues = new JavaScriptSerializer().Deserialize <AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
string[] Igonored = new JavaScriptSerializer().Deserialize <string[]>(ReportHelpers.Load(StorageAccount, "Configuration.WorkerJobToBeIgnored.json", ContainerName));
if (Igonored.Contains(job.JobInstanceName, StringComparer.OrdinalIgnoreCase))
{
continue;
}
if (faultRate > thresholdValues.WorkJobErrorThreshold && alert)
{
new SendAlertMailTask
{
AlertSubject = string.Format("Error: Alert for Nuget Work Service : {0} failure", job.JobInstanceName),
Details = string.Format("Rate of failure exceeded Error threshold for {0}. Threshold count : {1}%, failure in last 24 hour : {2}", job.JobInstanceName, thresholdValues.WorkJobErrorThreshold, faultCount),
AlertName = string.Format("Error: Nuget Work Service {0}", job.JobInstanceName),
Component = "Nuget Work Service",
Level = "Error"
}.ExecuteCommand();
}
else if (faultRate > thresholdValues.WorkJobWarningThreshold && alert)
{
new SendAlertMailTask
{
AlertSubject = string.Format("Warning: Alert for Nuget Work Service: {0} failure", job.JobInstanceName),
Details = string.Format("Rate of failure exceeded Warning threshold for {0}. Threshold count : {1}%, failure in last 24 hour : {2}", job.JobInstanceName, thresholdValues.WorkJobWarningThreshold, faultCount),
AlertName = string.Format("Warning: Nuget Work Service {0}", job.JobInstanceName),
Component = "Nuget Work Service",
Level = "Warning"
}.ExecuteCommand();
}
}
//check to make sure that the jobs that are not queued as part of scheduler are being invoked properly
if (invocationCount < ((lastNhour * 60 / job.FrequencyInMinutes) / 2))
{
new SendAlertMailTask
{
AlertSubject = string.Format("Error: Alert for Nuget Work Service : {0} failure", job.JobInstanceName),
Details = string.Format("In last 24 hours, invocation of {0} is only {1}, it's less than half of scheduled jobs", job.JobInstanceName, invocationCount),
AlertName = string.Format("Error: Nuget Work Service {0}", job.JobInstanceName),
Component = "Nuget Work Service",
Level = "Error"
}.ExecuteCommand();
}
}
List <WorkServiceAdmin> allkey = new List <WorkServiceAdmin>();
allkey.Add(new WorkServiceAdmin(WorkServiceUserName, WorkServiceAdminKey));
allkey.Add(new WorkServiceAdmin(WorkServiceUserName, WorkServiceFailoverAdminKey));
var json = new JavaScriptSerializer().Serialize(jobDetail);
var key = new JavaScriptSerializer().Serialize(allkey);
ReportHelpers.CreateBlob(StorageAccount, "WorkJobDetail.json", ContainerName, "application/json", ReportHelpers.ToStream(json));
ReportHelpers.CreateBlob(StorageAccount, "WorkServiceAdminKey.json", ContainerName, "application/json", ReportHelpers.ToStream(key));
}
private void CreatereportForSupportRequestsByCategory()
{
List <Tuple <string, string> > summary = new List <Tuple <string, string> >();
foreach (string category in supportRequestCategories)
{
summary.Add(new Tuple <string, string>(category, GetWorkItemsByTitle(category).ToString()));
}
JArray reportObject = ReportHelpers.GetJson(summary);
ReportHelpers.CreateBlob(StorageAccount, "SupportRequestsByCategoryReport.json", "dashboard", "application/json", ReportHelpers.ToStream(reportObject));
}