protected BasicLogFinder(SenseLogMasterBaseType logMasterBaseType, SenseLogBaseTypes senseLogBaseTypes, SenseLogSubTypes logSubType, DirectorySetting directory)
{
LogMasterBaseType = logMasterBaseType;
LogBaseTypes = senseLogBaseTypes;
LogSubTypes = logSubType;
Directory = directory;
}
public void LoadAndRead(DirectorySetting archivedLogFolder, LogFileDirectorSettings settings, FileMinerDto basicDataFromCase)
{
//Read root folder and one directory down.
_sessionData = new QvSessionData();
_basicDataFromCase = basicDataFromCase;
if (!archivedLogFolder.Exists)
{
Log.To.Main.Add($"Unable to read QV logs in rootFolder ('{archivedLogFolder}') is not a valid folder.");
return;
}
Initialize();
Log.To.Main.Add($"Reading log folder {archivedLogFolder.Path}");
var logs = EnumerateLogFiles(archivedLogFolder.Path, settings.StartDateForLogs)?.ToList() ?? new List <IFileInfo>();
if (!logs.Any())
{
Log.To.Main.Add($"No session files exists for {settings.StartDateForLogs} - {settings.StopDateForLogs} in log directory {archivedLogFolder}");
}
ReadLogs(logs, _sessionLogStartsWith, _sessionCols, settings);
_basicDataFromCase.TotalUniqueActiveApps = _sessionData.Apps.Count;
_basicDataFromCase.TotalUniqueActiveUsers = _sessionData.Users.Count;
_basicDataFromCase.TotalUniqueActiveUsersList = _sessionData.Users;
_basicDataFromCase.TotalUniqueActiveAppsList = _sessionData.Apps;
_basicDataFromCase.SessionLengthAvgInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Average() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.SessionLengthMedInMinutes = (int)Math.Round(_sessionData.SessionLenghts.Any() ? _sessionData.SessionLenghts.Median() : 0, MidpointRounding.AwayFromZero);
_basicDataFromCase.TotalNrOfSessions = _sessionData.SessionLenghts.Count;
}
private void CrawlAllLogBaseDirectories(DirectorySetting baseDir, string directoryName, Func <string, bool> ignorePaths)
{
Trace.WriteLine(baseDir.Path);
if (baseDir.Exists)
{
Crawler(directoryName, ignorePaths, baseDir.Path);
}
}
public NodeJsV1LogFolder(DirectorySetting directory, SenseLogBaseTypes senseLogBaseTypes, SenseLogSubTypes logSubType) : base(SenseLogMasterBaseType.NodeJsV1, senseLogBaseTypes, logSubType, directory)
{
//Severity= info
//Date =2015-10-15T15:12:37.304Z
//1444921956216_969c2051-f159-4bac-a233-33bab93e0798.log
//1444921956216_Global.log
//1444921956216_Request.log
//Is file within date span
}
public SenseLogBaseTypes GetSenseLogBaseTypes(DirectorySetting dir)
{
if (KnownLogFoldersAsString.Contains("," + dir.Name + ","))
{
SenseLogBaseTypes type;
if (Enum.TryParse(dir.Name, true, out type))
{
return(type);
}
}
return(SenseLogBaseTypes.Unknown);
}
public QlikViewSetup(
IEnumerable <QlikViewServiceInfo> serviceInfos,
DirectorySetting userDocDirectory,
DirectorySetting qvsLogDirectory,
FileSetting qvsEventLogFile,
FileSetting qvsSessionLogFile,
DirectorySetting qdsDirectory)
{
ServiceInfos = serviceInfos.ToArray();
UserDocDirectory = userDocDirectory;
QvsLogDirectory = qvsLogDirectory;
QvsEventLogFile = qvsEventLogFile;
QvsSessionLogFile = qvsSessionLogFile;
QdsDirectory = qdsDirectory;
}
private DirectorySetting CrawlAllLogBaseDirectories(DirectorySetting directory)
{
//Trace.WriteLine(directory.Path);
if (directory != null && directory.Exists)
{
if (IsBaseLogDirecory(directory))
{
foreach (string dir in directory.GetDirectories())
{
//todo:The _ignored files has dual purpose, and if there is folder called all it will be ignored.
if (_ignoredFiles.HasFlag(_folderFinder.GetSenseLogBaseTypes(new DirectorySetting(dir))))
{//_ignoredFiles != SenseLogBaseTypes.All &&
Trace.WriteLine("Ignoring directory=>" + dir);
continue;
}
var finder = new LogFileFinder(WriteFile, _fileSystem, _logger, null);
string localPath = _fileSystem.Path.Combine(_settings.OutputFolderPath, (_fileSystem.Path.GetFileName(directory.Path)).SanitizeFileName(), _fileSystem.Path.GetFileName(dir) + "");
//_fileSystem.EnsureDirectory(localPath);
finder.FindFilesInDirectories(new DirectorySetting(dir), _settings.StartDateForLogs, _settings.StopDateForLogs, localPath);
Trace.WriteLine("found=>" + dir);
}
}
var i = 0;
//FIX:on unstable networks this can return null
var dirs = _fileSystem.GetDirectories(directory.Path);
while (dirs == null)
{
dirs = _fileSystem.GetDirectories(directory.Path);
i++;
if (i > 10)
{
break;
}
}
foreach (string dir in dirs)
{
var baseDir = CrawlAllLogBaseDirectories(new DirectorySetting(dir));
if (baseDir != null)
{
return(baseDir);
}
}
}
return(null);
}
/// <summary>
/// Reads a directory item from the configuration node and returns a new instance of DirectorySetting.
/// </summary>
/// <param name="node">The XmlNode to read from.</param>
/// <returns>A DirectorySetting initialized with values read from the node.</returns>
protected DirectorySetting ReadDirectoryItem(XmlNode node)
{
// Create a DirectorySetting instance.
DirectorySetting Item = new DirectorySetting();
// Read the typical attributes.
ReadChildItem(node, Item);
// Check for a recurse attribute.
if (node.Attributes["recurse"] != null)
{
string Value = node.Attributes["recurse"].Value.ToLower();
Item.Recurse = (Value == "true" || Value == "yes" || Value == "on");
}
return(Item);
}
private void GetFileDirectories(DirectorySetting rootDir, DateTime from, DateTime to, string outputFilePath)
{
if (_ignorePaths != null)
{
if (!_ignorePaths.Invoke(rootDir.Name))
{
Trace.WriteLine($"Igoring=>{rootDir.Name}");
return;
}
}
GetFiles(rootDir.Path, from, to, outputFilePath);
_fileWriter(null, null);//dir counter hack.
foreach (string dir in rootDir.GetDirectories())
{
GetFileDirectories(new DirectorySetting(dir), from, to, _fileSystem.Path.Combine(outputFilePath, _fileSystem.Path.GetFileName(dir) + ""));
}
}
public BasicLogFinder Get(DirectorySetting dir)
{
var savedDir = dir;
var subType = GetSenseLogSubTypes(dir.Name);
if (subType != SenseLogSubTypes.Unknown)
{
dir = dir.ParentDirectory;//we've found him jim!
}
var type = GetSenseLogBaseTypes(dir);
if (type != SenseLogBaseTypes.Unknown)
{
return(Get(type, subType, savedDir));
}
return(null);
}
private DirectorySetting CrawlAllLogBaseDirectories(DirectorySetting directory)
{
if (directory != null && directory.Exists)
{
if (IsBaseLogDirecory(directory))
{
foreach (var dataMiner in _fileMiners)
{
var mineLocation = dataMiner.MineFromThisLocation(directory.Path, _fileSystem);
if (!string.IsNullOrEmpty(mineLocation))
{
ScanLogs(mineLocation, dataMiner);
}
}
}
var i = 0;
//FIX:on unstable networks this can return null
var dirs = _fileSystem.GetDirectories(directory.Path);
while (dirs == null)
{
dirs = _fileSystem.GetDirectories(directory.Path);
i++;
if (i > 10)
{
break;
}
}
foreach (string dir in dirs)
{
var baseDir = CrawlAllLogBaseDirectories(new DirectorySetting(dir));
if (baseDir != null)
{
return(baseDir);
}
}
}
return(null);
}
public ScriptV1LogFolder(DirectorySetting directory, SenseLogBaseTypes senseLogBaseTypes, SenseLogSubTypes logSubType) : base(SenseLogMasterBaseType.ScriptV1, senseLogBaseTypes, logSubType, directory)
{
}
private bool IsBaseLogDirecory(DirectorySetting directory)
{
return(directory.ChildExists("Repository\\Trace"));
}
//do something cool to remove this.. this is just a temp hack.. Im tired today..
//I did!!! finally 1 year afterwards...
public BasicLogFinder Get(SenseLogBaseTypes baseType, SenseLogSubTypes subType, DirectorySetting dir)
{
switch (baseType)
{
case SenseLogBaseTypes.AppMigration:
case SenseLogBaseTypes.Script:
case SenseLogBaseTypes.BrokerService:
case SenseLogBaseTypes.DataProfiling:
case SenseLogBaseTypes.HubService:
case SenseLogBaseTypes.AboutService:
case SenseLogBaseTypes.CapabilityService:
case SenseLogBaseTypes.ConnectorRegistryProxy:
case SenseLogBaseTypes.ConverterService:
case SenseLogBaseTypes.DepGraphService:
case SenseLogBaseTypes.DownloadPrepService:
case SenseLogBaseTypes.OdagService:
case SenseLogBaseTypes.WebExtensionService:
case SenseLogBaseTypes.HybridSetupConsoleBff:
case SenseLogBaseTypes.ResourceDistributionService:
case SenseLogBaseTypes.DeploymentBasedWarningsService:
//1444921956216_969c2051-f159-4bac-a233-33bab93e0798.log whoahhahah we are susceptible to the year 2286 problem...
//1444921956216_Global.log
//1444921956216_Request.log
return(new NodeJsV1LogFolder(dir, baseType, SenseLogSubTypes.RootFolder));
case SenseLogBaseTypes.Engine:
case SenseLogBaseTypes.Proxy:
case SenseLogBaseTypes.Printing:
case SenseLogBaseTypes.Repository:
case SenseLogBaseTypes.Scheduler:
case SenseLogBaseTypes.AppDistributionService: // Todo: Bugreport this log. Its just horrible.
case SenseLogBaseTypes.EntitlementProvisioningService: // Todo: Bugreport this log. Its just horrible.
case SenseLogBaseTypes.HybridDeploymentService: // Todo: Bugreport this log. Its just horrible.
switch (subType)
{
case SenseLogSubTypes.Audit:
return(new SenseV1LogFolder(dir, baseType, subType));
case SenseLogSubTypes.System:
return(new SenseV1LogFolder(dir, baseType, subType));
case SenseLogSubTypes.Trace:
return(new SenseV1LogFolder(dir, baseType, subType));
default:
return(new SenseV1LogFolder(dir, SenseLogBaseTypes.Unknown, subType));
}
default:
return(null);
}
}
public void FindFilesInDirectories(DirectorySetting rootDir, DateTime from, DateTime to, string outputFilePath)
{
GetFileDirectories(rootDir, from, to, outputFilePath);
}
public void Execute()
{
try
{
var qmsAddress = Settings.GetSetting($"{MonitorName}.QmsAddress", "(undefined)");
DirectorySetting archivedLogsLocation;
if (qmsAddress.Equals("(undefined)", StringComparison.InvariantCultureIgnoreCase))
{
qmsAddress = $"http://{(Dns.GetHostEntry(Dns.GetHostName()).HostName).ToLower()}:4799/QMS/Service";
}
string installationId;
License licence = null;
using (var qmsApiService = new QMS_API.AgentsQmsApiService(qmsAddress))
{
if (!qmsApiService.TestConnection())
{
Log.To.Main.Add($"Could not connect to QMS API {qmsAddress} in {MonitorName}");
return;
}
List <ServiceInfo> qvsServices = qmsApiService.GetServices(ServiceTypes.QlikViewServer);
QVSSettings qvsSettings = qmsApiService.GetQvsSettings(qvsServices[0].ID, QVSSettingsScope.Logging);
var folder = qvsSettings.Logging.Folder;
if (!Directory.Exists(folder))
{
Log.To.Main.Add($"The folder does not exist or we don't have access to the folder:'{folder}' will not read logs.");
return;
}
archivedLogsLocation = new DirectorySetting(folder);
var services = qmsApiService.GetServices(ServiceTypes.QlikViewServer | ServiceTypes.QlikViewDistributionService);
var qvServers = services.Where(p => p.Type == ServiceTypes.QlikViewServer).ToList();
installationId = qvServers.OrderBy(p => p.ID).First().ID.ToString();
qvServers.ForEach(p =>
{
licence = qmsApiService.GetLicense(p.Type == ServiceTypes.QlikViewServer ? LicenseType.QlikViewServer : LicenseType.Publisher, p.ID);
});
}
var logMinerData = new FileMinerDto();
var data = new StatisticsDto {
LogFileMinerData = logMinerData, CollectionDateUtc = logMinerData.CollectionDateUtc
};
var settings = new LogFileDirectorSettings
{
StartDateForLogs = DateTime.Now.AddDays(-2).Date,
StopDateForLogs = DateTime.Now.AddDays(-1).Date.AddMilliseconds(-1),
};
//settings.StartDateForLogs = DateTime.Parse("2019-04-30 00:00:00").AddDays(FAKERUNCOUNT);
//settings.StopDateForLogs = DateTime.Parse("2019-04-30 23:59:59").AddDays(FAKERUNCOUNT);
//archivedLogsLocation = new DirectorySetting(@"C:\ProgramData\QlikTech\QlikViewServer");
var logFileDirector = new QvLogDirector();
data.InstallationId = $"{licence?.Serial ?? "(unknown)"}_{installationId} ";
data.QlikViewLicence = _licenceHelper.AnalyzeLicense(licence);
data.LogFileMinerData.LicenseSerialNo = licence?.Serial ?? "(unknown qv)";
Log.To.Main.Add($"Starting log director on {archivedLogsLocation.Path}");
logFileDirector.LoadAndRead(archivedLogsLocation, settings, logMinerData);
Notify($"{MonitorName} has analyzed the following system", new List <string> {
JsonConvert.SerializeObject(data, Formatting.Indented)
}, "-1");
FAKERUNCOUNT++;
}
catch (Exception ex)
{
Log.To.Main.AddException($"Failed executing {MonitorName}", ex);
}
}
