internal bool AllRequiredInstancesHaveRunTheCollector(List <Instance> instancesToCollect)
{
if (instancesToCollect == null || instancesToCollect.Count == 0)
{
int numberOfLiveInstances = HeartBeatController.GetNumberOfLiveInstances();
Logger.LogDiagnostic("All instances need to run the collector");
return(numberOfLiveInstances <= NumberOfInstancesCollected);
}
var collectedInstances = GetCollectedInstances();
Logger.LogDiagnostic("Checking if Instances to collect list is subset of Already Collected instances");
if (instancesToCollect.All(collectedInstances.Contains))
{
Logger.LogDiagnostic("We done with collector on specified instances");
return(true);
}
var liveInstances = HeartBeatController.GetLiveInstances();
// If an instance is no longer live don't wait for it to run its collector
var instancesThatCanBeCollected = instancesToCollect.Intersect(liveInstances);
if (instancesThatCanBeCollected.Any(instanceToCollect => !collectedInstances.Contains(instanceToCollect)))
{
Logger.LogDiagnostic("Not all instances ran collector");
return(false);
}
Logger.LogDiagnostic("We done with collector on specified instances");
return(true);
}
public Task CancelSessionAsync()
{
LoadLatestUpdates(false);
Logger.LogSessionVerboseEvent($"Cancelling Session - {SessionId}. Session cancelled after { DateTime.UtcNow.Subtract(StartTime).TotalMinutes } minutes", SessionId.ToString());
List <Instance> InstancesToClean = new List <Instance>();
if (InstancesSpecified != null && InstancesSpecified.Count != 0)
{
InstancesToClean = InstancesSpecified;
}
else
{
InstancesToClean = HeartBeatController.GetLiveInstances().ToList();
}
foreach (var diagnoserSession in _diagnoserSessions)
{
if (diagnoserSession.AnalyzerStatus == DiagnosisStatus.Cancelled || diagnoserSession.CollectorStatus == DiagnosisStatus.Cancelled)
{
continue;
}
// Cancel anything that's not already complete
diagnoserSession.CollectorStatus = (DiagnosisStatus)Math.Max((int)diagnoserSession.CollectorStatus, (int)DiagnosisStatus.Cancelled);
diagnoserSession.AnalyzerStatus = (DiagnosisStatus)Math.Max((int)diagnoserSession.AnalyzerStatus, (int)DiagnosisStatus.Cancelled);
foreach (var instance in InstancesToClean)
{
CancelledInstance cancelled = new CancelledInstance
{
ProcessCleanupOnCancel = diagnoserSession.Diagnoser.ProcessCleanupOnCancel,
CancellationTime = DateTime.UtcNow,
DiagnoserName = diagnoserSession.Diagnoser.Name.Replace(" ", ""),
Name = instance.Name,
SessionId = SessionId.ToString()
};
var filePath = Path.Combine(Settings.CancelledDir, $"{cancelled.Name}.{cancelled.DiagnoserName}");
Logger.LogSessionVerboseEvent($"Creating file {filePath}", SessionId.ToString());
Infrastructure.Storage.SaveFile(cancelled.GetXmlStream(), filePath, StorageLocation.UserSiteData);
}
}
// Lets do this without a lock. If by any chance our
// locking logic has an issue, at-least we should cancel
// session properly
return(SaveUpdatesAsync(waitForLease: true));
}
internal bool ShouldCollectLogsOnInstance(Instance instance)
{
if (InstancesSpecified != null && InstancesSpecified.Count != 0)
{
Logger.LogDiagnostic(
InstancesSpecified.Contains(instance)
? "Yes we need to collect logs on instance {0}"
: "No we don't need to collect logs on instance {0}", instance.Name);
bool instanceStillValid = HeartBeatController.GetLiveInstances().Any(x => x.Name == instance.Name);
Logger.LogDiagnostic("Is the instance still a valid instance ? {0}", instanceStillValid);
return(instanceStillValid && InstancesSpecified.Contains(instance));
}
Logger.LogDiagnostic("Yes we need to collect logs on instance {0}", instance.Name);
return(true);
}
private void CheckAndReturnCollectorDetailedStatus(XElement diagnoserXml)
{
if (diagnoserXml.Parent != null)
{
var diagnosers = diagnoserXml.Parent;
if (diagnosers.Parent != null)
{
var sessionXml = diagnosers.Parent;
var timeRangeXml = sessionXml.Element(SessionXml.TimeRange);
var StartTime = DateTime.Parse(timeRangeXml.Element(SessionXml.StartTime).Value).ToUniversalTime();
var EndTime = DateTime.Parse(timeRangeXml.Element(SessionXml.EndTime).Value).ToUniversalTime();
string siteName = Infrastructure.Settings.SiteName;
if (siteName.Length > 10)
{
siteName = siteName.Substring(0, 10);
}
foreach (var instance in HeartBeatController.GetLiveInstances())
{
var path = Path.Combine("Logs",
siteName,
EndTime.ToString("yy-MM-dd"),
instance.Name,
this.Diagnoser.Collector.Name,
StartTime.ToString(SessionConstants.SessionFileNameFormat));
string fullDirPath = Path.Combine(@"d:\home\data\daas\", path);
var files = new List <string>();
if (Directory.Exists(fullDirPath))
{
files = Directory.GetFiles(fullDirPath, "diagstatus.diaglog", SearchOption.TopDirectoryOnly).ToList();
}
foreach (var statusFile in files)
{
try
{
using (FileStream fs = System.IO.File.Open(statusFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (StreamReader sr = new StreamReader(fs))
{
while (!sr.EndOfStream)
{
var msg = new DiagnoserStatusMessage
{
EntityType = instance.Name,
Message = sr.ReadLine()
};
CollectorStatusMessages.Add(msg);
}
}
}
}
catch (Exception ex)
{
Logger.LogErrorEvent($"Failed to read status file", ex);
}
}
}
}
}
}
static void Main(string[] args)
{
if (args.Length == 0)
{
ShowUsage();
return;
}
int argNum = 0;
while (argNum < args.Length)
{
Options option;
var currentArgument = args[argNum];
if (!ArguementIsAParameter(currentArgument))
{
ShowUsage();
return;
}
if (!Options.TryParse(args[argNum].Substring(1, args[argNum].Length - 1), true, out option))
{
ShowUsage();
return;
}
argNum++;
switch (option)
{
case (Options.ListSessions):
var allSessions = SessionController.GetAllSessions();
Console.WriteLine("All Sessions:");
foreach (var session in allSessions)
{
Console.WriteLine(" Session log path: " + session.FullPermanentStoragePath);
}
break;
case (Options.ListDiagnosers):
{
var diagnosers = SessionController.GetAllDiagnosers().ToList();
ListDiagnosers(diagnosers);
break;
}
case (Options.AnalyzeSession):
{
Logger.LogVerboseEvent($"DaasConsole AnalyzeSession started with {string.Join(" ", args)} parameters");
argNum = ModifySession(args, argNum, SessionController.Analyze);
break;
}
case (Options.CancelSession):
{
Logger.LogVerboseEvent($"DaasConsole CancelSession started with {string.Join(" ", args)} parameters");
argNum = ModifySession(args, argNum, SessionController.Cancel);
break;
}
case (Options.Setup):
{
SessionController.StartSessionRunner(sourceDir: ".", extraFilesToCopy: new List <string>()
{
@"Configuration\DiagnosticSettings.xml"
});
break;
}
case (Options.Help):
ShowUsage();
break;
case (Options.CollectLogs):
case (Options.Troubleshoot):
case (Options.CollectKillAnalyze):
{
CollectLogsAndTakeActions(option, args, ref argNum);
break;
}
case (Options.GetSasUri):
Console.WriteLine("Sas Uri:");
Console.WriteLine(SessionController.BlobStorageSasUri);
break;
case (Options.SetSasUri):
string uri = args[argNum];
argNum++;
SessionController.BlobStorageSasUri = uri;
Console.WriteLine("Sas Uri is now:");
Console.WriteLine(SessionController.BlobStorageSasUri);
break;
case (Options.GetSetting):
{
string settingName;
try
{
settingName = args[argNum];
argNum++;
}
catch
{
Console.WriteLine("GetSetting options not correctly specified");
ShowUsage();
return;
}
var settingValue = settings.GetSetting(settingName);
Console.WriteLine("Got {0} = {1}", settingName, settingValue);
break;
}
case (Options.SetSetting):
{
string settingName;
string settingValue;
try
{
settingName = args[argNum];
argNum++;
settingValue = args[argNum];
argNum++;
}
catch
{
Console.WriteLine("SetSetting options not correctly specified");
ShowUsage();
return;
}
settings.SaveSetting(settingName, settingValue);
settingValue = settings.GetSetting(settingName);
Console.WriteLine("Set {0} = {1}", settingName, settingValue);
break;
}
case (Options.ListInstances):
{
Console.WriteLine("Current Instances:");
foreach (var instance in HeartBeatController.GetLiveInstances())
{
Console.WriteLine(instance);
}
break;
}
default:
break;
}
Console.WriteLine();
}
}