internal bool AllRequiredInstancesHaveRunTheCollector(List <Instance> instancesToCollect)
{
if (instancesToCollect == null || instancesToCollect.Count == 0)
{
int numberOfLiveInstances = HeartBeatController.GetNumberOfLiveInstances();
Logger.LogDiagnostic("All instances need to run the collector");
return(numberOfLiveInstances <= NumberOfInstancesCollected);
}
var collectedInstances = GetCollectedInstances();
Logger.LogDiagnostic("Checking if Instances to collect list is subset of Already Collected instances");
if (instancesToCollect.All(collectedInstances.Contains))
{
Logger.LogDiagnostic("We done with collector on specified instances");
return(true);
}
var liveInstances = HeartBeatController.GetLiveInstances();
// If an instance is no longer live don't wait for it to run its collector
var instancesThatCanBeCollected = instancesToCollect.Intersect(liveInstances);
if (instancesThatCanBeCollected.Any(instanceToCollect => !collectedInstances.Contains(instanceToCollect)))
{
Logger.LogDiagnostic("Not all instances ran collector");
return(false);
}
Logger.LogDiagnostic("We done with collector on specified instances");
return(true);
}
public Task CancelSessionAsync()
{
LoadLatestUpdates(false);
Logger.LogSessionVerboseEvent($"Cancelling Session - {SessionId}. Session cancelled after { DateTime.UtcNow.Subtract(StartTime).TotalMinutes } minutes", SessionId.ToString());
List <Instance> InstancesToClean = new List <Instance>();
if (InstancesSpecified != null && InstancesSpecified.Count != 0)
{
InstancesToClean = InstancesSpecified;
}
else
{
InstancesToClean = HeartBeatController.GetLiveInstances().ToList();
}
foreach (var diagnoserSession in _diagnoserSessions)
{
if (diagnoserSession.AnalyzerStatus == DiagnosisStatus.Cancelled || diagnoserSession.CollectorStatus == DiagnosisStatus.Cancelled)
{
continue;
}
// Cancel anything that's not already complete
diagnoserSession.CollectorStatus = (DiagnosisStatus)Math.Max((int)diagnoserSession.CollectorStatus, (int)DiagnosisStatus.Cancelled);
diagnoserSession.AnalyzerStatus = (DiagnosisStatus)Math.Max((int)diagnoserSession.AnalyzerStatus, (int)DiagnosisStatus.Cancelled);
foreach (var instance in InstancesToClean)
{
CancelledInstance cancelled = new CancelledInstance
{
ProcessCleanupOnCancel = diagnoserSession.Diagnoser.ProcessCleanupOnCancel,
CancellationTime = DateTime.UtcNow,
DiagnoserName = diagnoserSession.Diagnoser.Name.Replace(" ", ""),
Name = instance.Name,
SessionId = SessionId.ToString()
};
var filePath = Path.Combine(Settings.CancelledDir, $"{cancelled.Name}.{cancelled.DiagnoserName}");
Logger.LogSessionVerboseEvent($"Creating file {filePath}", SessionId.ToString());
Infrastructure.Storage.SaveFile(cancelled.GetXmlStream(), filePath, StorageLocation.UserSiteData);
}
}
// Lets do this without a lock. If by any chance our
// locking logic has an issue, at-least we should cancel
// session properly
return(SaveUpdatesAsync(waitForLease: true));
}
internal bool ShouldCollectLogsOnInstance(Instance instance)
{
if (InstancesSpecified != null && InstancesSpecified.Count != 0)
{
Logger.LogDiagnostic(
InstancesSpecified.Contains(instance)
? "Yes we need to collect logs on instance {0}"
: "No we don't need to collect logs on instance {0}", instance.Name);
bool instanceStillValid = HeartBeatController.GetLiveInstances().Any(x => x.Name == instance.Name);
Logger.LogDiagnostic("Is the instance still a valid instance ? {0}", instanceStillValid);
return(instanceStillValid && InstancesSpecified.Contains(instance));
}
Logger.LogDiagnostic("Yes we need to collect logs on instance {0}", instance.Name);
return(true);
}
private static void SendHeartBeat()
{
if (VerbosityLevel >= Verbosity.Information)
{
Logger.LogDiagnostic("Sending Heartbeat...");
}
HeartBeatController.SendHeartBeat();
Logger.LogDiagnostic("Sent heartbeat");
// No need to bother cleaning out stale heartbeats all the time.
cleanOutHeartBeats++;
if (cleanOutHeartBeats >= 5)
{
Logger.LogDiagnostic("Cleaning out stale heartbeats (if there are any)");
HeartBeatController.DeleteExpiredHeartBeats();
cleanOutHeartBeats = 0;
}
else
{
Logger.LogDiagnostic("Clean heart beat counter: " + cleanOutHeartBeats);
}
}
public MegaClass(
UserLicensesController a2,
TimeEntrySuggestedRowsController a3,
WorkHourSuggestedRowsController a4,
KpiFormulasController a5,
UserSettingsController a6,
SecurityController a7,
SoapApiKeysController a8,
NotificationSettingsController a9,
FlextimeAdjustmentsController a10,
AuthorizedIpAddressesController a11,
BankAccountsController a12,
ApiClientsController a13,
OrganizationSettingsController a14,
LogosController a15,
ProjectMemberCostExceptionsController a16,
OrganizationDetailsController a17,
AddonsController a18,
TravelExpenseCountrySettingsController a19,
ProductCountrySettingsController a20,
CustomerCountrySettingsController a21,
KpiComparisonController a22,
TravelExpenseReceiptsController a23,
TravelReimbursementStatusController a24,
TravelReimbursementsController a25,
ResourcingOverviewController a26,
TermsOfServiceApprovalsController a27,
FinancialsController a28,
CalendarGroupMembersController a29,
ActivityParticipantsController a30,
ActivitiesController a31,
PermissionProfilesController a32,
CalendarGroupsController a33,
ResourcesController a34,
DemoDataController a35,
LinksController a36,
ReimbursedWorkHoursController a37,
ReimbursedProjectTravelExpensesController a38,
ReimbursedProjectFeesController a39,
ProjectsOverviewController a40,
ContactRolesController a41,
CustomerMarketSegmentsController a42,
ProjectTotalFeesController a43,
BillingInformationUpdateController a44,
KeywordsController a45,
FlatRatesController a46,
BusinessOverviewController a47,
SpecialUserOptionsController a48,
UninvoicedProjectsController a49,
TeamProductivityController a50,
ProjectBillingCustomersController a51,
MarketSegmentsController a52,
ProjectProductsController a53,
ScheduleOverviewController a54,
SharedDashboardAccessRightProfilesController a55,
SharedDashboardsController a56,
InvoicesOverviewController a57,
ProposalBillingPlanController a58,
SalesOverviewController a59,
CustomersOverviewController a60,
ProposalProjectPlanController a61,
TimeEntryTypesController a62,
CommunicationTypesController a63,
ContactCommunicationsController a64,
FlextimeController a65,
ProjectForecastsController a66,
ResourceAllocationsController a67,
TemporaryProjectFeesController a68,
InvoiceTemplateSettingsController a69,
TemporaryProjectTravelExpensesController a70,
TemporaryWorkHoursController a71,
InvoiceTemplatesController a72,
WorkdaysController a73,
InvoiceSettingsController a74,
OrganizationsController a75,
ProjectTaskStatusesController a76,
ActivityTypesController a77,
AddressesController a78,
DashboardPartsController a79,
DashboardWithPartsController a80,
DashboardsController a81,
InvoiceRowsController a82,
InvoicesController a83,
HolidaysController a84,
PermissionsController a85,
QuickSearchController a86,
ProposalTemplatesController a87,
InvoiceTotalsController a88,
ProposalTotalsController a89,
ProposalWorkhoursController a90,
ProposalSubtotalsController a91,
ProposalFeesController a92,
ReportsController a93,
ProposalStatusesController a94,
InvoiceStatusesController a95,
ProposalsController a96,
StatusHistoryController a97,
PhaseStatusTypesController a98,
CostCentersController a99,
ProjectWorktypesController a100,
PricelistVersionsController a101,
OvertimePricesController a102,
AllUsersController a103,
TimeEntriesController a104,
WorkTypesController a105,
WorkHoursController a106,
ProjectWorkHourPricesController a107,
TravelPricesController a108,
WorkHourPricesController a109,
ProductPricesController a110,
ProjectRecurringFeeRulesController a111,
ProjectTravelExpensesController a112,
TravelExpensesController a113,
ProjectFeesController a114,
SalesReceivableAccountsController a115,
ValueAddedTaxesController a116,
ProductsController a117,
BusinessUnitsController a118,
CollaborationNotesController a119,
ContactsController a120,
CurrencyBasesController a121,
CurrenciesController a122,
CountriesController a123,
CustomersController a124,
FileDataController a125,
FilesController a126,
FormattingCulturesController a127,
IndustriesController a128,
LanguagesController a129,
LeadSourcesController a130,
MenuController a131,
PersonalSettingsController a132,
PhaseMembersController a133,
PhasesController a134,
PricelistsController a135,
OvertimesController a136,
ProjectsController a137,
ProjectStatusTypesController a138,
SalesAccountsController a139,
SalesNotesController a140,
ProductCategoriesController a141,
WorkContractsController a142,
SalesStatusController a143,
SalesStatusTypeController a144,
TimeZonesController a145,
UsersController a146,
ScheduledWorkJobsController a147,
PasswordChangeController a148,
UserInactivationInformationController a149,
EmailAddressChangeController a150,
EmailController a151,
PdfController a152,
BearerAuthenticationController a153,
ExternalAuthenticationController a154,
HeartBeatController a155
)
{
_a2 = _a2;
_a3 = _a3;
_a4 = _a4;
_a5 = _a5;
_a6 = _a6;
_a7 = _a7;
_a8 = _a8;
_a9 = _a9;
_a10 = _a10;
_a11 = _a11;
_a12 = _a12;
_a13 = _a13;
_a14 = _a14;
_a15 = _a15;
_a16 = _a16;
_a17 = _a17;
_a18 = _a18;
_a19 = _a19;
_a20 = _a20;
_a21 = _a21;
_a22 = _a22;
_a23 = _a23;
_a24 = _a24;
_a25 = _a25;
_a26 = _a26;
_a27 = _a27;
_a28 = _a28;
_a29 = _a29;
_a30 = _a30;
_a31 = _a31;
_a32 = _a32;
_a33 = _a33;
_a34 = _a34;
_a35 = _a35;
_a36 = _a36;
_a37 = _a37;
_a38 = _a38;
_a39 = _a39;
_a40 = _a40;
_a41 = _a41;
_a42 = _a42;
_a43 = _a43;
_a44 = _a44;
_a45 = _a45;
_a46 = _a46;
_a47 = _a47;
_a48 = _a48;
_a49 = _a49;
_a50 = _a50;
_a51 = _a51;
_a52 = _a52;
_a53 = _a53;
_a54 = _a54;
_a55 = _a55;
_a56 = _a56;
_a57 = _a57;
_a58 = _a58;
_a59 = _a59;
_a60 = _a60;
_a61 = _a61;
_a62 = _a62;
_a63 = _a63;
_a64 = _a64;
_a65 = _a65;
_a66 = _a66;
_a67 = _a67;
_a68 = _a68;
_a69 = _a69;
_a70 = _a70;
_a71 = _a71;
_a72 = _a72;
_a73 = _a73;
_a74 = _a74;
_a75 = _a75;
_a76 = _a76;
_a77 = _a77;
_a78 = _a78;
_a79 = _a79;
_a80 = _a80;
_a81 = _a81;
_a82 = _a82;
_a83 = _a83;
_a84 = _a84;
_a85 = _a85;
_a86 = _a86;
_a87 = _a87;
_a88 = _a88;
_a89 = _a89;
_a90 = _a90;
_a91 = _a91;
_a92 = _a92;
_a93 = _a93;
_a94 = _a94;
_a95 = _a95;
_a96 = _a96;
_a97 = _a97;
_a98 = _a98;
_a99 = _a99;
_a100 = _a100;
_a101 = _a101;
_a102 = _a102;
_a103 = _a103;
_a104 = _a104;
_a105 = _a105;
_a106 = _a106;
_a107 = _a107;
_a108 = _a108;
_a109 = _a109;
_a110 = _a110;
_a111 = _a111;
_a112 = _a112;
_a113 = _a113;
_a114 = _a114;
_a115 = _a115;
_a116 = _a116;
_a117 = _a117;
_a118 = _a118;
_a119 = _a119;
_a120 = _a120;
_a121 = _a121;
_a122 = _a122;
_a123 = _a123;
_a124 = _a124;
_a125 = _a125;
_a126 = _a126;
_a127 = _a127;
_a128 = _a128;
_a129 = _a129;
_a130 = _a130;
_a131 = _a131;
_a132 = _a132;
_a133 = _a133;
_a134 = _a134;
_a135 = _a135;
_a136 = _a136;
_a137 = _a137;
_a138 = _a138;
_a139 = _a139;
_a140 = _a140;
_a141 = _a141;
_a142 = _a142;
_a143 = _a143;
_a144 = _a144;
_a145 = _a145;
_a146 = _a146;
_a147 = _a147;
_a148 = _a148;
_a149 = _a149;
_a150 = _a150;
_a151 = _a151;
_a152 = _a152;
_a153 = _a153;
_a154 = _a154;
_a155 = _a155;
}
private static void StartSessionRunner()
{
int logCounter = 0;
sleepIntervalForHeartbeatCheck = (sleepIntervalForHeartbeatCheck < _DaaS.FrequencyToCheckForNewSessionsAt.TotalSeconds) ? _DaaS.FrequencyToCheckForNewSessionsAt.TotalSeconds : sleepIntervalForHeartbeatCheck;
_lastInstanceCountCheck = DateTime.UtcNow;
while (true)
{
Logger.LogDiagnostic("Checking for active sessions...");
var activeSessions = _DaaS.GetAllActiveSessions().ToList();
if (activeSessions.Count == 0)
{
Logger.LogDiagnostic("No active sessions");
var cancelledInstances = _DaaS.GetCancelledInstances();
if (cancelledInstances != null && cancelledInstances.Count > 0)
{
Logger.LogDiagnostic($"Found {cancelledInstances.Count} cancelled instances with instance names = { string.Join(",", cancelledInstances.Select(x => x.Name)) }");
var anyInstanceToCancel = cancelledInstances.FirstOrDefault();
if (anyInstanceToCancel != null)
{
_DaaS.CancelActiveSessionOnThisInstance(anyInstanceToCancel.SessionId);
}
RemoveOldCancelledFiles(cancelledInstances);
KillChildProcessesIfAnyForCancelledInstances(cancelledInstances);
}
}
else
{
// ensure that we don't end up submitting too many sessions
// This can happen if AutoHealing has generated too many files
int activeSessionCount = activeSessions.Count;
if (activeSessionCount > 5)
{
foreach (var session in activeSessions.Skip(5))
{
Logger.LogSessionVerboseEvent($"Deleting Session [{ session.SessionId }] because there are more than { activeSessionCount } sessions active", session.SessionId.ToString());
try
{
var task = Task.Run(async() => { await _DaaS.Delete(session, true); });
task.Wait();
}
catch (Exception ex)
{
Logger.LogSessionErrorEvent("Failed while deleting Session", ex, session.SessionId.ToString());
}
}
}
}
foreach (var session in activeSessions)
{
Logger.LogInfo(" Session " + session.SessionId + " - Status: " + session.Status);
Logger.LogInfo(" " + session.FullPermanentStoragePath);
}
try
{
_DaaS.RunActiveSessions();
if (DateTime.UtcNow.Subtract(_lastHeartbeatSent).TotalSeconds > sleepIntervalForHeartbeatCheck)
{
_lastHeartbeatSent = DateTime.UtcNow;
SendHeartBeat();
}
}
catch (Exception e)
{
Logger.LogErrorEvent("Encountered unhandled exception while running sessions", e);
}
if (DateTime.UtcNow.Subtract(_lastInstanceCountCheck).TotalMinutes > InstanceCountCheckFrequency)
{
_lastInstanceCountCheck = DateTime.UtcNow;
try
{
logCounter++;
int instanceCount = HeartBeatController.GetNumberOfLiveInstances();
sleepIntervalForHeartbeatCheck = GetSleepIntervalBetweenHeartbeats(instanceCount);
// Treat sleepInterval as minutes for InstanceCountCheckFrequency
// to avoid making this call every 30 minutes
InstanceCountCheckFrequency = Convert.ToInt32(sleepIntervalForHeartbeatCheck);
if (logCounter == 5)
{
logCounter = 0;
Logger.LogVerboseEvent($"Live Instance Count = {instanceCount} and sleepInterval between heartbeats = {sleepIntervalForHeartbeatCheck}");
}
}
catch (Exception)
{
}
}
Logger.LogDiagnostic("Finished iteration");
Thread.Sleep(_DaaS.FrequencyToCheckForNewSessionsAt);
}
}
private void CheckAndReturnCollectorDetailedStatus(XElement diagnoserXml)
{
if (diagnoserXml.Parent != null)
{
var diagnosers = diagnoserXml.Parent;
if (diagnosers.Parent != null)
{
var sessionXml = diagnosers.Parent;
var timeRangeXml = sessionXml.Element(SessionXml.TimeRange);
var StartTime = DateTime.Parse(timeRangeXml.Element(SessionXml.StartTime).Value).ToUniversalTime();
var EndTime = DateTime.Parse(timeRangeXml.Element(SessionXml.EndTime).Value).ToUniversalTime();
string siteName = Infrastructure.Settings.SiteName;
if (siteName.Length > 10)
{
siteName = siteName.Substring(0, 10);
}
foreach (var instance in HeartBeatController.GetLiveInstances())
{
var path = Path.Combine("Logs",
siteName,
EndTime.ToString("yy-MM-dd"),
instance.Name,
this.Diagnoser.Collector.Name,
StartTime.ToString(SessionConstants.SessionFileNameFormat));
string fullDirPath = Path.Combine(@"d:\home\data\daas\", path);
var files = new List <string>();
if (Directory.Exists(fullDirPath))
{
files = Directory.GetFiles(fullDirPath, "diagstatus.diaglog", SearchOption.TopDirectoryOnly).ToList();
}
foreach (var statusFile in files)
{
try
{
using (FileStream fs = System.IO.File.Open(statusFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (StreamReader sr = new StreamReader(fs))
{
while (!sr.EndOfStream)
{
var msg = new DiagnoserStatusMessage
{
EntityType = instance.Name,
Message = sr.ReadLine()
};
CollectorStatusMessages.Add(msg);
}
}
}
}
catch (Exception ex)
{
Logger.LogErrorEvent($"Failed to read status file", ex);
}
}
}
}
}
}
private int NormalizeCountPerInstance(int count)
{
return(count / HeartBeatController.GetNumberOfLiveInstances());
}
private void Start()
{
heartBeatController = FindObjectOfType <HeartBeatController>();
}
public HeartBeatControllerTests()
{
_hbr = new HeartBeatRepository();
_hbc = new HeartBeatController(_hbr);
}
public HeartBeatControllerTests()
{
_hbr = new HeartBeatRepository();
_hbc = new HeartBeatController(_hbr);
}
static void Main(string[] args)
{
if (args.Length == 0)
{
ShowUsage();
return;
}
int argNum = 0;
while (argNum < args.Length)
{
Options option;
var currentArgument = args[argNum];
if (!ArguementIsAParameter(currentArgument))
{
ShowUsage();
return;
}
if (!Options.TryParse(args[argNum].Substring(1, args[argNum].Length - 1), true, out option))
{
ShowUsage();
return;
}
argNum++;
switch (option)
{
case (Options.ListSessions):
var allSessions = SessionController.GetAllSessions();
Console.WriteLine("All Sessions:");
foreach (var session in allSessions)
{
Console.WriteLine(" Session log path: " + session.FullPermanentStoragePath);
}
break;
case (Options.ListDiagnosers):
{
var diagnosers = SessionController.GetAllDiagnosers().ToList();
ListDiagnosers(diagnosers);
break;
}
case (Options.AnalyzeSession):
{
Logger.LogVerboseEvent($"DaasConsole AnalyzeSession started with {string.Join(" ", args)} parameters");
argNum = ModifySession(args, argNum, SessionController.Analyze);
break;
}
case (Options.CancelSession):
{
Logger.LogVerboseEvent($"DaasConsole CancelSession started with {string.Join(" ", args)} parameters");
argNum = ModifySession(args, argNum, SessionController.Cancel);
break;
}
case (Options.Setup):
{
SessionController.StartSessionRunner(sourceDir: ".", extraFilesToCopy: new List <string>()
{
@"Configuration\DiagnosticSettings.xml"
});
break;
}
case (Options.Help):
ShowUsage();
break;
case (Options.CollectLogs):
case (Options.Troubleshoot):
case (Options.CollectKillAnalyze):
{
CollectLogsAndTakeActions(option, args, ref argNum);
break;
}
case (Options.GetSasUri):
Console.WriteLine("Sas Uri:");
Console.WriteLine(SessionController.BlobStorageSasUri);
break;
case (Options.SetSasUri):
string uri = args[argNum];
argNum++;
SessionController.BlobStorageSasUri = uri;
Console.WriteLine("Sas Uri is now:");
Console.WriteLine(SessionController.BlobStorageSasUri);
break;
case (Options.GetSetting):
{
string settingName;
try
{
settingName = args[argNum];
argNum++;
}
catch
{
Console.WriteLine("GetSetting options not correctly specified");
ShowUsage();
return;
}
var settingValue = settings.GetSetting(settingName);
Console.WriteLine("Got {0} = {1}", settingName, settingValue);
break;
}
case (Options.SetSetting):
{
string settingName;
string settingValue;
try
{
settingName = args[argNum];
argNum++;
settingValue = args[argNum];
argNum++;
}
catch
{
Console.WriteLine("SetSetting options not correctly specified");
ShowUsage();
return;
}
settings.SaveSetting(settingName, settingValue);
settingValue = settings.GetSetting(settingName);
Console.WriteLine("Set {0} = {1}", settingName, settingValue);
break;
}
case (Options.ListInstances):
{
Console.WriteLine("Current Instances:");
foreach (var instance in HeartBeatController.GetLiveInstances())
{
Console.WriteLine(instance);
}
break;
}
default:
break;
}
Console.WriteLine();
}
}