public void LoadAndRead(DirectorySetting archivedLogFolder, LogFileDirectorSettings settings, FileMinerDto basicDataFromCase)
{
//Read root folder and one directory down.
_sessionData = new QvSessionData();
_basicDataFromCase = basicDataFromCase;
if (!archivedLogFolder.Exists)
{
Log.To.Main.Add($"Unable to read QV logs in rootFolder ('{archivedLogFolder}') is not a valid folder.");
return;
}
Initialize();
Log.To.Main.Add($"Reading log folder {archivedLogFolder.Path}");
var logs = EnumerateLogFiles(archivedLogFolder.Path, settings.StartDateForLogs)?.ToList() ?? new List <IFileInfo>();
if (!logs.Any())
{
Log.To.Main.Add($"No session files exists for {settings.StartDateForLogs} - {settings.StopDateForLogs} in log directory {archivedLogFolder}");
}
ReadLogs(logs, _sessionLogStartsWith, _sessionCols, settings);
_basicDataFromCase.TotalUniqueActiveApps = _sessionData.Apps.Count;
_basicDataFromCase.TotalUniqueActiveUsers = _sessionData.Users.Count;
_basicDataFromCase.TotalUniqueActiveUsersList = _sessionData.Users;
_basicDataFromCase.TotalUniqueActiveAppsList = _sessionData.Apps;
_basicDataFromCase.SessionLengthAvgInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Average() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.SessionLengthMedInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Median() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.TotalNrOfSessions = _sessionData.SessionLenghts.Count;
}
public void GetItDone()
{
var txtData = new TextData();
var installationId = "f113051f-ef10-4969-a2b1-e3c890506b4e";
var monitorNameFile = "SenseLogFileParserMonitor";
var monitorNameStats = "SenseStatisticsMonitor";
var date = DateTime.Now.AddMonths(-12);
var gjallarhornDb = new GjallarhornDb(FileSystem.Singleton);
gjallarhornDb.EnsureMonitorTableExists(monitorNameFile);
gjallarhornDb.EnsureMonitorTableExists(monitorNameStats);
Random random = new Random();
while (date < DateTime.Now)
{
date = date.AddHours(1);
var dto = new FileMinerDto();
dto.LicenseSerialNo = "(fake LICENCE)";
dto.CollectionDateUtc = date;
dto.IsMonthly = false;
var data = new StatisticsDto {
LogFileMinerData = dto, CollectionDateUtc = dto.CollectionDateUtc
};
data.QLikSenseCalInfo = JsonConvert.DeserializeObject <QLikSenseCalInfo>(txtData.GetCalInfo());
data.InstallationId = "(fake)_" + installationId;
dto.TotalUniqueActiveAppsList = new Dictionary <string, int>();
gjallarhornDb.SaveMonitorData(monitorNameFile, JsonConvert.SerializeObject(data));
if (date.Hour == 0)
{
var dtoDay = new FileMinerDto();
dtoDay.LicenseSerialNo = "(fake)";
dtoDay.CollectionDateUtc = date;
dto.TotalUniqueActiveApps = random.Next(100);
dto.TotalUniqueActiveUsers = random.Next(55);
dto.SessionLengthAvgInMinutes = random.Next(100);
dto.SessionLengthMedInMinutes = random.Next(50);
var dataDay = new StatisticsDto {
LogFileMinerData = dtoDay, CollectionDateUtc = dtoDay.CollectionDateUtc
};
dataDay.InstallationId = "(fake)_" + installationId;
dtoDay.TotalUniqueActiveAppsList = new Dictionary <string, int>();
dataDay.QlikSenseMachineInfos = JsonConvert.DeserializeObject <List <QlikSenseMachineInfo> >(txtData.GetMachineInfo());
dataDay.QlikSenseServiceInfo = JsonConvert.DeserializeObject <List <QlikSenseServiceInfo> >(txtData.GetServiceInfo());
dataDay.QlikSenseQrsAbout = JsonConvert.DeserializeObject <QlikSenseQrsAbout>(txtData.GetQrsAbout());
dataDay.QlikSenseAppListShort = JsonConvert.DeserializeObject <List <QlikSenseAppListShort> >(txtData.GetAppList());
gjallarhornDb.SaveMonitorData(monitorNameStats, JsonConvert.SerializeObject(dataDay));
}
}
}
private void FinalizeStatistics(FileMinerDto data, Stopwatch stopwatch)
{
if (data == null)
{
return;
}
data.TotalUniqueActiveUsers = data.TotalUniqueActiveUsersList?.Count ?? -1;
data.TotalUniqueActiveApps = data.TotalUniqueActiveAppsList?.Count ?? -1;
_fileMinerData.TotalNrOfFiles = _localFileCounter;
_fileMinerData.TotalNrOfDirectories = _localDirCounter;
_fileMinerData.TotalScanTimeTakenSec = stopwatch.Elapsed.Seconds;
}
public void LoadAndRead(DirectorySetting[] directories, LogFileDirectorSettings settings, FileMinerDto fileMinerData)
{
_settings = settings;
_fileMinerData = fileMinerData;
_fileMiners = new ActiveFileMiners().GetQlikSenseFileMiners();
var timer = Stopwatch.StartNew();
foreach (DirectorySetting directory in directories)
{
CrawlAllLogBaseDirectories(directory);
}
foreach (var dataMiner in _fileMiners)
{
dataMiner.FinaliseStatistics();
}
timer.Stop();
FinalizeStatistics(fileMinerData, timer);
}
public void Execute()
{
try
{
var logFileDirector = new LogFileDirector(FileSystem.Singleton);
var logMinerData = new FileMinerDto();
var data = new StatisticsDto {
LogFileMinerData = logMinerData, CollectionDateUtc = logMinerData.CollectionDateUtc
};
string archivedLogsLocation;
//default we ask sense for the settings needed.
if (String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.OverideLogFilePath", "")) &&
String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.LicenseSerialNo", "")) &&
String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.ServiceClusterId", "")))
{
var host = Settings.GetSetting($"{MonitorName}.HostName", "(undefined)");
if (host.Equals("(undefined)", StringComparison.InvariantCultureIgnoreCase))
{
host = (Dns.GetHostEntry(Dns.GetHostName()).HostName).ToLower();
}
var senseApi = SenseApiSupport.Create(host);
var helper = new SenseApiHelper();
SenseEnums senseEnums = new SenseEnums(senseApi);
try { data.QlikSenseMachineInfos = helper.GetQlikSenseMachineInfos(senseApi, senseEnums).ToList(); } catch (Exception e) { data.Exceptions.Add(e); }
try { data.QlikSenseLicenseAgent = helper.ExecuteLicenseAgent(senseApi, senseEnums); } catch (Exception e) { data.Exceptions.Add(e); }
try { data.QlikSenseServiceInfo = helper.GetQlikSenseServiceInfos(senseApi, senseEnums).ToList(); } catch (Exception e) { data.Exceptions.Add(e); }
archivedLogsLocation = helper.GetQlikSenseArchivedFolderLocation(senseApi);
_installationId = $"{data.QlikSenseLicenseAgent?.LicenseSerialNo ?? "(unknown)"}_{data.QlikSenseServiceInfo?.FirstOrDefault()?.ServiceClusterId.ToString() ?? "(unknown)"} ";
_licenseSerialNr = data.QlikSenseLicenseAgent?.LicenseSerialNo ?? "(unknown)";
data.QlikSenseLicenseAgent = null;
data.QlikSenseServiceInfo = null;
data.QlikSenseMachineInfos = null;
}
else // pull from settings
{
_licenseSerialNr = Settings.GetSetting($"{MonitorName}.LicenseSerialNo", "");
_installationId = $"{_licenseSerialNr}_{Settings.GetSetting($"{MonitorName}.ServiceClusterId", "")}";
archivedLogsLocation = Settings.GetSetting($"{MonitorName}.OverideLogFilePath", "");
}
data.InstallationId = _installationId;
data.LogFileMinerData.LicenseSerialNo = _licenseSerialNr;
//string archivedLogsLocation = @"C:\temp\ArchivedLogs";//@"D:\SFDCData\files\01471384\SenseCollector_e8e2d1bc-3c1e-41d7-9a9c-0cac78c7539d\SenseCollector_e8e2d1bc-3c1e-41d7-9a9c-0cac78c7539d";
//get yesterday +1
var settings = new LogFileDirectorSettings
{
StartDateForLogs = DateTime.Now.AddDays(-2).Date,
StopDateForLogs = DateTime.Now.AddDays(-1).Date.AddMilliseconds(-1),
};
//settings.StartDateForLogs = DateTime.Parse("2018-08-27 00:00:00");
//settings.StopDateForLogs = DateTime.Parse("2018-08-27 23:59:59");
settings.StartDateForLogs = DateTime.Parse("2019-03-09 00:00:00").AddDays(FAKERUNCOUNT);
settings.StopDateForLogs = DateTime.Parse("2019-03-09 23:59:59").AddDays(FAKERUNCOUNT);
logFileDirector.LoadAndRead(new[] { new DirectorySetting(archivedLogsLocation) }, settings, logMinerData);
//persisting current days apps and users for more analysis.
var db = new GjallarhornDb(FileSystem.Singleton);
CheckMontlySending(settings.StartDateForLogs.Month);
db.AddToMontlyStats(logMinerData.TotalUniqueActiveAppsList, settings.StartDateForLogs.Year, settings.StartDateForLogs.Month, MontlyStatsType.Apps);
db.AddToMontlyStats(logMinerData.TotalUniqueActiveUsersList, settings.StartDateForLogs.Year, settings.StartDateForLogs.Month, MontlyStatsType.Users);
Trace.WriteLine($"{settings.StartDateForLogs.ToString("yyyy-MM-dd")} sessionCount=>{data.LogFileMinerData.TotalNrOfSessions} on FakeRun:{FAKERUNCOUNT}");
Notify($"{MonitorName} has analyzed the following system", new List <string> {
JsonConvert.SerializeObject(data, Formatting.Indented)
}, "-1");
FAKERUNCOUNT++;
}
catch (Exception ex)
{
Log.To.Main.AddException($"Failed executing {MonitorName}", ex);
}
}
public void Execute()
{
try
{
var qmsAddress = Settings.GetSetting($"{MonitorName}.QmsAddress", "(undefined)");
DirectorySetting archivedLogsLocation;
if (qmsAddress.Equals("(undefined)", StringComparison.InvariantCultureIgnoreCase))
{
qmsAddress = $"http://{(Dns.GetHostEntry(Dns.GetHostName()).HostName).ToLower()}:4799/QMS/Service";
}
string installationId;
License licence = null;
using (var qmsApiService = new QMS_API.AgentsQmsApiService(qmsAddress))
{
if (!qmsApiService.TestConnection())
{
Log.To.Main.Add($"Could not connect to QMS API {qmsAddress} in {MonitorName}");
return;
}
List <ServiceInfo> qvsServices = qmsApiService.GetServices(ServiceTypes.QlikViewServer);
QVSSettings qvsSettings = qmsApiService.GetQvsSettings(qvsServices[0].ID, QVSSettingsScope.Logging);
var folder = qvsSettings.Logging.Folder;
if (!Directory.Exists(folder))
{
Log.To.Main.Add($"The folder does not exist or we don't have access to the folder:'{folder}' will not read logs.");
return;
}
archivedLogsLocation = new DirectorySetting(folder);
var services = qmsApiService.GetServices(ServiceTypes.QlikViewServer | ServiceTypes.QlikViewDistributionService);
var qvServers = services.Where(p => p.Type == ServiceTypes.QlikViewServer).ToList();
installationId = qvServers.OrderBy(p => p.ID).First().ID.ToString();
qvServers.ForEach(p =>
{
licence = qmsApiService.GetLicense(p.Type == ServiceTypes.QlikViewServer ? LicenseType.QlikViewServer : LicenseType.Publisher, p.ID);
});
}
var logMinerData = new FileMinerDto();
var data = new StatisticsDto {
LogFileMinerData = logMinerData, CollectionDateUtc = logMinerData.CollectionDateUtc
};
var settings = new LogFileDirectorSettings
{
StartDateForLogs = DateTime.Now.AddDays(-2).Date,
StopDateForLogs = DateTime.Now.AddDays(-1).Date.AddMilliseconds(-1),
};
//settings.StartDateForLogs = DateTime.Parse("2019-04-30 00:00:00").AddDays(FAKERUNCOUNT);
//settings.StopDateForLogs = DateTime.Parse("2019-04-30 23:59:59").AddDays(FAKERUNCOUNT);
//archivedLogsLocation = new DirectorySetting(@"C:\ProgramData\QlikTech\QlikViewServer");
var logFileDirector = new QvLogDirector();
data.InstallationId = $"{licence?.Serial ?? "(unknown)"}_{installationId} ";
data.QlikViewLicence = _licenceHelper.AnalyzeLicense(licence);
data.LogFileMinerData.LicenseSerialNo = licence?.Serial ?? "(unknown qv)";
Log.To.Main.Add($"Starting log director on {archivedLogsLocation.Path}");
logFileDirector.LoadAndRead(archivedLogsLocation, settings, logMinerData);
Notify($"{MonitorName} has analyzed the following system", new List <string> {
JsonConvert.SerializeObject(data, Formatting.Indented)
}, "-1");
FAKERUNCOUNT++;
}
catch (Exception ex)
{
Log.To.Main.AddException($"Failed executing {MonitorName}", ex);
}
}
