public void LoadAndRead(DirectorySetting archivedLogFolder, LogFileDirectorSettings settings, FileMinerDto basicDataFromCase)
{
//Read root folder and one directory down.
_sessionData = new QvSessionData();
_basicDataFromCase = basicDataFromCase;
if (!archivedLogFolder.Exists)
{
Log.To.Main.Add($"Unable to read QV logs in rootFolder ('{archivedLogFolder}') is not a valid folder.");
return;
}
Initialize();
Log.To.Main.Add($"Reading log folder {archivedLogFolder.Path}");
var logs = EnumerateLogFiles(archivedLogFolder.Path, settings.StartDateForLogs)?.ToList() ?? new List <IFileInfo>();
if (!logs.Any())
{
Log.To.Main.Add($"No session files exists for {settings.StartDateForLogs} - {settings.StopDateForLogs} in log directory {archivedLogFolder}");
}
ReadLogs(logs, _sessionLogStartsWith, _sessionCols, settings);
_basicDataFromCase.TotalUniqueActiveApps = _sessionData.Apps.Count;
_basicDataFromCase.TotalUniqueActiveUsers = _sessionData.Users.Count;
_basicDataFromCase.TotalUniqueActiveUsersList = _sessionData.Users;
_basicDataFromCase.TotalUniqueActiveAppsList = _sessionData.Apps;
_basicDataFromCase.SessionLengthAvgInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Average() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.SessionLengthMedInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Median() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.TotalNrOfSessions = _sessionData.SessionLenghts.Count;
}
private void ReadLog(IFileInfo file, List <ColumnInfo> masterCols, LogFileDirectorSettings settings)
{
Log.To.Main.Add($"Reading log {file.FullName}");
var nrOfFailedLogLines = 0;
var nrOfFailedLogLinesDate = 0;
var expectedMinimumColCount = 19;
var lineCounter = 0;
try
{
{
Dictionary <string, ColumnInfo> cols;
using (var fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096))
{
using (var sr = new StreamReader(fs))
{
if (sr.EndOfStream)
{
return; //empty file
}
var headers = sr.ReadLine();
cols = Validate(headers, masterCols);
if (cols == null)
{
Log.To.Main.Add($"ReadLog failed with evaluating headers headers: {headers}. Ignoring file: {file}. ");
return;
}
while (!sr.EndOfStream)
{
lineCounter++;
var line = sr.ReadLine() + "";
var a = line.Split('\t');
if (a.Length < expectedMinimumColCount)
{
nrOfFailedLogLines++;
continue;
}
var timestamp = TryParse(a[cols["timestamp"].Index], DateTime.MinValue);
if (timestamp == DateTime.MinValue)
{
nrOfFailedLogLinesDate++;
continue;
}
if (timestamp.Date < settings.StartDateForLogs.Date)
{
continue;//inefficient but effective. If we store postion we need to get the private variables from the streamreader or create our own.
}
//we should not continue reading into the day that is already active.
if (timestamp.Date > settings.StopDateForLogs.Date)
{
Trace.WriteLine($"stopping reading log due to date {timestamp}");
break;
}
var user = a[cols["Authenticated user"].Index];
if (_sessionData.Users.ContainsKey(user))
{
_sessionData.Users[user]++;
}
else
{
_sessionData.Users[user] = 1;
}
var docs = a[cols["Document"].Index];
if (_sessionData.Apps.ContainsKey(docs))
{
_sessionData.Apps[docs]++;
}
else
{
_sessionData.Apps[docs] = 1;
}
if (TimeSpan.TryParseExact((a[cols["Session Duration"].Index]), @"hh\:mm\:ss", CultureInfo.InvariantCulture, out var timespan))
{
_sessionData.SessionLenghts.Add((int)timespan.TotalMinutes);
}
//QvsLogAgentDatapointSet dps = QvsLogAgentDatapointSetTemplate.CreateDatapointSet();
//foreach (ColumnInfo c in cols.Where(c => a.Length > c.Index))
//{
// //dps.AddDatapoint(c.DatapointName, a[c.Index]);
//}
//AddDatapointBatch(dps);
}
}
}
}
if (nrOfFailedLogLines > 0 || nrOfFailedLogLinesDate > 0)
{
Log.To.Main.Add($"Failed reading some lines in log: {file}. Nr of lines skipped: {nrOfFailedLogLines}. Datetime failures are :{nrOfFailedLogLinesDate} and lines read {lineCounter}");
}
}
catch (Exception ex)
{
Log.To.Main.Add($"Failed reading and sending log: {file} on position: {lineCounter}. {ex}");
}
}
private void ReadLogs(IEnumerable <IFileInfo> logFiles, string startsWith, List <ColumnInfo> columns, LogFileDirectorSettings settings)
{
foreach (var file in logFiles)
{
if (file.Name.Contains(startsWith, StringComparison.InvariantCultureIgnoreCase))
{
ReadLog(file, columns, settings);
}
}
}
public void Execute()
{
try
{
var logFileDirector = new LogFileDirector(FileSystem.Singleton);
var logMinerData = new FileMinerDto();
var data = new StatisticsDto {
LogFileMinerData = logMinerData, CollectionDateUtc = logMinerData.CollectionDateUtc
};
string archivedLogsLocation;
//default we ask sense for the settings needed.
if (String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.OverideLogFilePath", "")) &&
String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.LicenseSerialNo", "")) &&
String.IsNullOrWhiteSpace(Settings.GetSetting($"{MonitorName}.ServiceClusterId", "")))
{
var host = Settings.GetSetting($"{MonitorName}.HostName", "(undefined)");
if (host.Equals("(undefined)", StringComparison.InvariantCultureIgnoreCase))
{
host = (Dns.GetHostEntry(Dns.GetHostName()).HostName).ToLower();
}
var senseApi = SenseApiSupport.Create(host);
var helper = new SenseApiHelper();
SenseEnums senseEnums = new SenseEnums(senseApi);
try { data.QlikSenseMachineInfos = helper.GetQlikSenseMachineInfos(senseApi, senseEnums).ToList(); } catch (Exception e) { data.Exceptions.Add(e); }
try { data.QlikSenseLicenseAgent = helper.ExecuteLicenseAgent(senseApi, senseEnums); } catch (Exception e) { data.Exceptions.Add(e); }
try { data.QlikSenseServiceInfo = helper.GetQlikSenseServiceInfos(senseApi, senseEnums).ToList(); } catch (Exception e) { data.Exceptions.Add(e); }
archivedLogsLocation = helper.GetQlikSenseArchivedFolderLocation(senseApi);
_installationId = $"{data.QlikSenseLicenseAgent?.LicenseSerialNo ?? "(unknown)"}_{data.QlikSenseServiceInfo?.FirstOrDefault()?.ServiceClusterId.ToString() ?? "(unknown)"} ";
_licenseSerialNr = data.QlikSenseLicenseAgent?.LicenseSerialNo ?? "(unknown)";
data.QlikSenseLicenseAgent = null;
data.QlikSenseServiceInfo = null;
data.QlikSenseMachineInfos = null;
}
else // pull from settings
{
_licenseSerialNr = Settings.GetSetting($"{MonitorName}.LicenseSerialNo", "");
_installationId = $"{_licenseSerialNr}_{Settings.GetSetting($"{MonitorName}.ServiceClusterId", "")}";
archivedLogsLocation = Settings.GetSetting($"{MonitorName}.OverideLogFilePath", "");
}
data.InstallationId = _installationId;
data.LogFileMinerData.LicenseSerialNo = _licenseSerialNr;
//string archivedLogsLocation = @"C:\temp\ArchivedLogs";//@"D:\SFDCData\files\01471384\SenseCollector_e8e2d1bc-3c1e-41d7-9a9c-0cac78c7539d\SenseCollector_e8e2d1bc-3c1e-41d7-9a9c-0cac78c7539d";
//get yesterday +1
var settings = new LogFileDirectorSettings
{
StartDateForLogs = DateTime.Now.AddDays(-2).Date,
StopDateForLogs = DateTime.Now.AddDays(-1).Date.AddMilliseconds(-1),
};
//settings.StartDateForLogs = DateTime.Parse("2018-08-27 00:00:00");
//settings.StopDateForLogs = DateTime.Parse("2018-08-27 23:59:59");
settings.StartDateForLogs = DateTime.Parse("2019-03-09 00:00:00").AddDays(FAKERUNCOUNT);
settings.StopDateForLogs = DateTime.Parse("2019-03-09 23:59:59").AddDays(FAKERUNCOUNT);
logFileDirector.LoadAndRead(new[] { new DirectorySetting(archivedLogsLocation) }, settings, logMinerData);
//persisting current days apps and users for more analysis.
var db = new GjallarhornDb(FileSystem.Singleton);
CheckMontlySending(settings.StartDateForLogs.Month);
db.AddToMontlyStats(logMinerData.TotalUniqueActiveAppsList, settings.StartDateForLogs.Year, settings.StartDateForLogs.Month, MontlyStatsType.Apps);
db.AddToMontlyStats(logMinerData.TotalUniqueActiveUsersList, settings.StartDateForLogs.Year, settings.StartDateForLogs.Month, MontlyStatsType.Users);
Trace.WriteLine($"{settings.StartDateForLogs.ToString("yyyy-MM-dd")} sessionCount=>{data.LogFileMinerData.TotalNrOfSessions} on FakeRun:{FAKERUNCOUNT}");
Notify($"{MonitorName} has analyzed the following system", new List <string> {
JsonConvert.SerializeObject(data, Formatting.Indented)
}, "-1");
FAKERUNCOUNT++;
}
catch (Exception ex)
{
Log.To.Main.AddException($"Failed executing {MonitorName}", ex);
}
}
public void Execute()
{
try
{
var qmsAddress = Settings.GetSetting($"{MonitorName}.QmsAddress", "(undefined)");
DirectorySetting archivedLogsLocation;
if (qmsAddress.Equals("(undefined)", StringComparison.InvariantCultureIgnoreCase))
{
qmsAddress = $"http://{(Dns.GetHostEntry(Dns.GetHostName()).HostName).ToLower()}:4799/QMS/Service";
}
string installationId;
License licence = null;
using (var qmsApiService = new QMS_API.AgentsQmsApiService(qmsAddress))
{
if (!qmsApiService.TestConnection())
{
Log.To.Main.Add($"Could not connect to QMS API {qmsAddress} in {MonitorName}");
return;
}
List <ServiceInfo> qvsServices = qmsApiService.GetServices(ServiceTypes.QlikViewServer);
QVSSettings qvsSettings = qmsApiService.GetQvsSettings(qvsServices[0].ID, QVSSettingsScope.Logging);
var folder = qvsSettings.Logging.Folder;
if (!Directory.Exists(folder))
{
Log.To.Main.Add($"The folder does not exist or we don't have access to the folder:'{folder}' will not read logs.");
return;
}
archivedLogsLocation = new DirectorySetting(folder);
var services = qmsApiService.GetServices(ServiceTypes.QlikViewServer | ServiceTypes.QlikViewDistributionService);
var qvServers = services.Where(p => p.Type == ServiceTypes.QlikViewServer).ToList();
installationId = qvServers.OrderBy(p => p.ID).First().ID.ToString();
qvServers.ForEach(p =>
{
licence = qmsApiService.GetLicense(p.Type == ServiceTypes.QlikViewServer ? LicenseType.QlikViewServer : LicenseType.Publisher, p.ID);
});
}
var logMinerData = new FileMinerDto();
var data = new StatisticsDto {
LogFileMinerData = logMinerData, CollectionDateUtc = logMinerData.CollectionDateUtc
};
var settings = new LogFileDirectorSettings
{
StartDateForLogs = DateTime.Now.AddDays(-2).Date,
StopDateForLogs = DateTime.Now.AddDays(-1).Date.AddMilliseconds(-1),
};
//settings.StartDateForLogs = DateTime.Parse("2019-04-30 00:00:00").AddDays(FAKERUNCOUNT);
//settings.StopDateForLogs = DateTime.Parse("2019-04-30 23:59:59").AddDays(FAKERUNCOUNT);
//archivedLogsLocation = new DirectorySetting(@"C:\ProgramData\QlikTech\QlikViewServer");
var logFileDirector = new QvLogDirector();
data.InstallationId = $"{licence?.Serial ?? "(unknown)"}_{installationId} ";
data.QlikViewLicence = _licenceHelper.AnalyzeLicense(licence);
data.LogFileMinerData.LicenseSerialNo = licence?.Serial ?? "(unknown qv)";
Log.To.Main.Add($"Starting log director on {archivedLogsLocation.Path}");
logFileDirector.LoadAndRead(archivedLogsLocation, settings, logMinerData);
Notify($"{MonitorName} has analyzed the following system", new List <string> {
JsonConvert.SerializeObject(data, Formatting.Indented)
}, "-1");
FAKERUNCOUNT++;
}
catch (Exception ex)
{
Log.To.Main.AddException($"Failed executing {MonitorName}", ex);
}
}