private void ExecuteAllActionsForVMSS(ARMDeployment dep)
{
int instanceId = TryConvertInstanceNameToInstanceId(this.VMName);
// TODO instead of using 0, take 5 random and use them
instanceId = instanceId == -1 ? 0 : instanceId;
Task <string> modelTask = null;
SALsA.GetInstance(Id).ICM.QueueICMDiscussion(dep.ToString());
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerConsoleSerialOutputFilename, GenevaActions.GetVMConsoleSerialLogs(Id, dep, instanceId), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerVMScreenshotOutputFilename, GenevaActions.GetVMConsoleScreenshot(Id, dep, instanceId), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerVMModelAndViewOutputFilename, modelTask = GenevaActions.GetVMModelAndInstanceView(Id, dep, instanceId), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerInspectIaaSDiskOutputFilename, GenevaActions.InspectIaaSDiskForARMVM(Id, dep, instanceId), Id)
);
var rawInfo = LogContainerId(modelTask, Id);
if (rawInfo != null)
{
GlobalInfo.Update(
new Guid(rawInfo.ContainerId),
new Guid(rawInfo.NodeId),
rawInfo.Cluster
);
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByContainerId(Id, GlobalInfo), Id)
);
}
}
private void GetAllNodeDiagnosticsFiles(string cluster, string nodeid, string startTime, string endTime)
{
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerHostMultiFilename,
GenevaActions.GetNodeDiagnosticsFiles(Id, cluster, nodeid, String.Join(",", Constants.GetNodeDiagnosticsFilesTagsParamMultiHost), startTime, endTime), Id)
);
}
/*
* private bool AnalyzeHost()
* {
* var currentICM = SALsA.GetInstance(Id).ICM;
* var title = currentICM.CurrentICM.Title;
* var isHostIssue = Regex.Match(title, @"HostGAPlugin.*Cluster.*Node.*(\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}", RegexOptions.IgnoreCase).Success;
* if (isHostIssue)
* {
* var splitTitle = title.ToLowerInvariant().Replace(" :", ":").Replace(": ", ":").Replace(",", " ").Replace(".", " ").Replace("nodeid", "node").Split(' ');
* splitTitle = Array.FindAll(splitTitle, s => s.Contains(":"));
* var dict = splitTitle.ToDictionary(
* k => k.Split(':')[0],
* e => e.Split(':')[1]
* );
* var cluster = dict["cluster"];
* var nodeid = dict["node"];
* var creationTime = currentICM.CurrentICM.CreateDate;
* var startTime = creationTime.AddHours(-12);
* var endTime = new DateTime(Math.Min(creationTime.AddHours(+12).Ticks, DateTime.UtcNow.Ticks));
* SALsA.GetInstance(Id).TaskManager.AddTask(
* Utility.SaveAndSendBlobTask(Constants.AnalyzerHostGAPluginFilename,
* GenevaActions.GetNodeDiagnosticsFiles(Id, cluster, nodeid, startTime.ToString("s"), endTime.ToString("s")), Id)
* );
* }
* return isHostIssue;
* }
*/
private void ExecuteAllActionsForIaaS(ARMDeployment dep)
{
Task <string> modelTask = null;
SALsA.GetInstance(Id).ICM.QueueICMDiscussion(dep.ToString());
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerConsoleSerialOutputFilename, GenevaActions.GetVMConsoleSerialLogs(Id, dep), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerVMScreenshotOutputFilename, GenevaActions.GetVMConsoleScreenshot(Id, dep), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerVMModelAndViewOutputFilename, modelTask = GenevaActions.GetVMModelAndInstanceView(Id, dep), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerInspectIaaSDiskOutputFilename, GenevaActions.InspectIaaSDiskForARMVM(Id, dep), Id)
);
var rawInfo = LogContainerId(modelTask, Id);
if (rawInfo != null)
{
GlobalInfo.Update(
new Guid(rawInfo.ContainerId),
new Guid(rawInfo.NodeId),
rawInfo.Cluster
);
var startTime = SALsA.GetInstance(Id).ICM.ICMImpactStartTime().AddHours(-12);
var endTime = new DateTime(Math.Min(startTime.AddHours(+24).Ticks, DateTime.UtcNow.Ticks));
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByContainerId(Id, GlobalInfo), Id)
);
}
}
private void AnalyzerInternal(bool checkARM = true, bool checkRDFE = true)
{
(var arm, var rdfe) = CallARMAndRDFE(checkARM, checkRDFE);
(var type, var dep) = DetectVMType(arm, rdfe);
if (dep == null && IsCustomRun == false)
{
try
{
var instance = AnalyzeARMResourceURI(SubscriptionId.ToString(), ResourceGroupName, VMName);
GlobalInfo.Update(
new Guid(instance.Last().ContainerId),
new Guid(instance.Last().NodeId),
instance.Last().Cluster
);
if (!string.IsNullOrWhiteSpace(instance.FirstOrDefault().Usage_ResourceGroupName))
{
this.ResourceGroupName = instance.FirstOrDefault().Usage_ResourceGroupName;
}
this.VMName = instance.FirstOrDefault().RoleInstanceName;
}
catch
{
try
{
var instance = AnalyzeRDFEResourceURI(SubscriptionId.ToString(), ResourceGroupName, VMName);
GlobalInfo.Update(
new Guid(instance.Last().ContainerId),
new Guid(instance.Last().NodeId),
instance.Last().Cluster
);
this.ResourceGroupName = instance.FirstOrDefault().TenantId;
this.VMName = instance.FirstOrDefault().RoleInstanceName;
}
catch
{
SALsA.GetInstance(Id).State = SALsAState.NotFound;
throw new Exception("VM not found");
}
}
(type, dep) = DetectVMType(arm, rdfe);
if (dep == null && IsCustomRun == false)
{
SALsA.GetInstance(Id).ICM.QueueICMDiscussion(String.Format("Could not find VM: {0} in RG: {1}. This VM might have been already deleted or moved",
this.VMName, this.ResourceGroupName));
// Lets try to check kusto data
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByContainerId(Id, GlobalInfo), Id));
}
}
CallInternalComputeTypes(type, dep);
}
private void RefreshTempPage(bool isTemp = false)
{
if (SALsA.GetInstance(Id).State == SALsAState.Ignore)
{
return; // Ignore the ICM
}
lock (__lockObj)
{
var message = Utility.GenerateICMHTMLPage(Id, MessageQueue.ToArray(), Log.StartTime, isTemp);
SAS = BlobStorageUtility.UploadICMRun(Id, message);
SALsA.GetInstance(Id)?.RefreshTable();
}
}
public static void Run(int icm, object manualRun = null)
{
try
{
SALsA.AddInstance(icm);
SALsA.GetInstance(icm).ICM.PostICMHeader();
// We do not need to keep the analyzer in memory, for now.
_ = manualRun == null ? new Analyzer(icm) : new Analyzer(icm, manualRun);
SALsA.GetInstance(icm)?.TaskManager.WaitAllTasks();
}
catch (Exception ex)
{
Log.Critical("Main failed !");
Log.Exception(ex);
SALsA.GetInstance(icm)?.TaskManager.WaitAllTasks();
try
{
Log.Information("ICM State : {0}", Utility.ObjectToJson(SALsA.GetInstance(icm).ICM));
}
catch { }
try
{
Log.Information("ICM SALsA : {0}", Utility.ObjectToJson(SALsA.GetInstance(icm)));
}
catch { }
//throw ex;
}
finally
{
SALsA.GetInstance(icm)?.ICM.EmptyMessageQueue();
if (SALsA.GetInstance(icm)?.State == SALsAState.Running)
{
if (SALsA.GetInstance(icm)?.ICM?.SAS != null)
{
SALsA.GetInstance(icm).State = SALsAState.Done;
}
else
{
SALsA.GetInstance(icm).State = SALsAState.UnknownException;
}
}
BlobStorageUtility.UploadLog(icm);
SALsA.GetInstance(icm)?.RefreshTable();
}
}
private void ExecuteAllActionsForPaaS(RDFEDeployment dep)
{
var instance = dep.RoleInstances.Where(x => String.Equals(x.RoleInstanceName, VMName, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (instance?.RoleInstanceName == null)
{
instance = dep.RoleInstances.Where(x => String.Equals(x.RoleInstanceName, TryConvertInstanceNameToVMName(this.VMName), StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
}
if (instance?.RoleInstanceName == null)
{
instance = dep.RoleInstances.Where(x => String.Equals(x.ID.ToString(), VMName)).FirstOrDefault();
}
GlobalInfo = new ShortRDFERoleInstance
{
Icm = Id,
Fabric = dep.FabricGeoId,
DeploymentId = dep.Id,
DeploymentName = dep.Name,
ContainerID = instance.ID,
InstanceName = instance.RoleInstanceName
};
Task <string> modelTask = null;
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerVMScreenshotOutputFilename, GenevaActions.GetClassicVMConsoleScreenshot(Id, GlobalInfo), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByDeploymentIdorVMName(Id, GlobalInfo), Id),
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerContainerSettings, modelTask = GenevaActions.GetContainerSettings(Id, GlobalInfo), Id)
);
try
{
var model = Utility.JsonToObject <Json2Class.ContainerSettings>(modelTask.Result);
GlobalInfo.NodeId = new Guid(model.NodeId);
}
catch (Exception ex)
{
Log.Critical("Failed to populate NodeId");
Log.Exception(ex);
}
finally
{
SALsA.GetInstance(Id)?.ICM.QueueICMDiscussion(GlobalInfo.ToString());
}
}
public static async Task SaveAndSendBlobTask(string name, Task <Image> task, int Id)
{
name = Utility.FormatFileName(Id, name);
var output = await task;
if (output == null)
{
Log.Warning("Could not create File {0} because Task output was null", name);
return;
}
using (MemoryStream ms = new MemoryStream())
{
output.Save(ms, ImageFormat.Png);
await BlobStorage.UploadBytes(Id, name, ms.ToArray(), "image/png");
}
SendSASToICM(name, Id);
BlobStorageUtility.SaveToFile(name, output, Id);
}
public Analyzer(int Id, object manualRun)
{
this.IsCustomRun = true;
this.Id = Id;
StartTime = SALsA.GetInstance(Id).ICM.CurrentICM.CreateDate;
Log.Information("Received ManualRun order type {0} with param {1} : ",
manualRun.GetType(), Utility.ObjectToJson(manualRun));
if (manualRun.GetType() == typeof(ManualRun_ICM))
{
ManualRun_ICM manualArm = (ManualRun_ICM)manualRun;
SubscriptionId = manualArm.SubscriptionID;
ResourceGroupName = manualArm.ResourceGroupName;
VMName = manualArm.VMName;
AnalyzerInternal();
}
else if (manualRun.GetType() == typeof(ManualRun_IID))
{
ManualRun_IID manualIID = (ManualRun_IID)manualRun;
SubscriptionId = manualIID.SubscriptionID;
ResourceGroupName = manualIID.ResourceGroupName;
VMName = manualIID.VMName;
ARMDeployment dep = null;
int instanceId = -1;
if (String.IsNullOrEmpty(manualIID.Region))
{
Log.Information("Calling automatic ARM VMdetection. No Region parameter provided.");
ARMSubscription arm = AnalyzeARMSubscription(SubscriptionId, this.ResourceGroupName);
dep = AnalyzeARMDeployment(arm);
instanceId = TryConvertInstanceNameToInstanceId(this.VMName);
}
else
{
dep = new ARMDeployment();
dep.Name = manualIID.VMName;
dep.Subscriptions = manualIID.SubscriptionID.ToString();
dep.ResourceGroups = manualIID.ResourceGroupName;
dep.Location = manualIID.Region;
instanceId = manualIID.Instance;
}
if (instanceId < 0)
{
Log.Information("No Instance ID detected. Assuming this is a normal single IaaS VM");
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(
Constants.AnalyzerInspectIaaSDiskOutputFilename,
GenevaActions.InspectIaaSDiskForARMVM(Id, dep), Id));
}
else
{
Log.Information("No Instance ID detected. Assuming this is a normal single IaaS VM");
SALsA.GetInstance(Id).TaskManager.AddTask(
BlobStorageUtility.SaveAndSendBlobTask(
Constants.AnalyzerInspectIaaSDiskOutputFilename,
GenevaActions.InspectIaaSDiskForARMVM(Id, dep, instanceId), Id));
}
}
else if (manualRun.GetType() == typeof(ManualRun_RDFE_Fabric))
{
var rdfe = (ManualRun_RDFE_Fabric)manualRun;
var vmInfo = new ShortRDFERoleInstance
{
ContainerID = new Guid(rdfe.ContainerID),
Fabric = rdfe.FabricCluster,
NodeId = new Guid(rdfe.NodeId)
};
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByContainerId(Id, vmInfo), Id).Wait();
}
else if (manualRun.GetType() == typeof(ManualRun_RDFE_Tenant))
{
var rdfe = (ManualRun_RDFE_Tenant)manualRun;
var vmInfo = new ShortRDFERoleInstance
{
Fabric = rdfe.FabricCluster,
DeploymentId = rdfe.DeploymentID,
InstanceName = rdfe.RoleInstanceName
};
BlobStorageUtility.SaveAndSendBlobTask(Constants.AnalyzerNodeDiagnosticsFilename, GenevaActions.GetNodeDiagnosticsFilesByDeploymentIdorVMName(Id, vmInfo), Id).Wait();
}
}