private ProcessLogTypeResult ProcessZip(LogSetInfo logSetInfo, LogTypeInfo logTypeInfo, IList <IPlugin> plugins)
{
var processingStatistics = new ProcessLogTypeResult();
using var zip = ZipFile.Open(logSetInfo.Path, ZipArchiveMode.Read);
foreach (var fileEntry in zip.Entries)
{
if (!logTypeInfo.FileBelongsToThisType(fileEntry.FullName))
{
continue;
}
var fileNameWithPrefix = string.IsNullOrWhiteSpace(logSetInfo.Prefix)
? fileEntry.FullName
: $"{logSetInfo.Prefix}/{fileEntry.FullName}";
_logger.LogInformation("Processing file {logFile}", fileNameWithPrefix);
var fileTimer = Stopwatch.StartNew();
var processFileResult = ProcessZippedFile(fileEntry, fileNameWithPrefix, logTypeInfo, plugins);
processingStatistics.AddProcessingInfo(fileTimer.Elapsed, fileEntry.Length, processFileResult);
if (!processFileResult.IsSuccessful)
{
break;
}
LogFileProcessingResults(fileNameWithPrefix, fileEntry.Length, processFileResult, fileTimer.Elapsed);
}
return(processingStatistics);
}
private ProcessLogTypeResult ProcessDir(LogSetInfo logSetInfo, LogTypeInfo logTypeInfo, IList <IPlugin> plugins)
{
var processingStatistics = new ProcessLogTypeResult();
foreach (var filePath in Directory.EnumerateFiles(logSetInfo.Path, "*", SearchOption.AllDirectories))
{
var normalizedPath = filePath.NormalizePath(logSetInfo.Path);
if (!logTypeInfo.FileBelongsToThisType(normalizedPath))
{
continue;
}
_logger.LogInformation("Processing file {}", normalizedPath);
var fileSizeBytes = new FileInfo(filePath).Length;
var fileTimer = Stopwatch.StartNew();
var processFileResult = ProcessFile(filePath, normalizedPath, logTypeInfo, plugins);
processingStatistics.AddProcessingInfo(fileTimer.Elapsed, fileSizeBytes, processFileResult);
if (!processFileResult.IsSuccessful)
{
break;
}
LogFileProcessingResults(normalizedPath, fileSizeBytes, processFileResult, fileTimer.Elapsed);
}
return(processingStatistics);
}
private (string, ExitReason) GetExitReasonAndErrorMessageIfApplicable(ProcessLogTypeResult failedProcessLogTypeResult, Dictionary <LogType, ProcessLogTypeResult> logProcessingStatistics)
{
if (failedProcessLogTypeResult != null)
{
return(failedProcessLogTypeResult.ErrorMessage, failedProcessLogTypeResult.ExitReason);
}
var processedAnyFiles = logProcessingStatistics.Any(kvp => kvp.Value.FilesProcessed > 0);
return(processedAnyFiles
? (null, ExitReason.CompletedSuccessfully)
: (FailureReasonMessageGenerator.NoTableauLogFilesFound(_config.RequestedPlugins, _config.OriginalLocation), ExitReason.LogSetDoesNotContainRelevantLogs));
}
private ProcessLogTypeResult ProcessLogType(IEnumerable <LogSetInfo> logSetParts, LogTypeInfo logTypeInfo, IList <IPlugin> plugins)
{
var overallProcessingNumbers = new ProcessLogTypeResult();
using var _ = _logger.BeginScope(logTypeInfo.LogType);
foreach (var logSetInfo in logSetParts)
{
_logger.LogInformation("Starting to process log set part `{logSetPartPath}` for {logType} logs", logSetInfo.Path, logTypeInfo.LogType);
var partProcessingResults = logSetInfo.IsZip
? ProcessZip(logSetInfo, logTypeInfo, plugins)
: ProcessDir(logSetInfo, logTypeInfo, plugins);
_logger.LogInformation("Completed processing `{logSetPartPath}` for {logType} logs. {partProcessingResults}", logSetInfo.Path, logTypeInfo.LogType, partProcessingResults);
overallProcessingNumbers.AddNumbersFrom(partProcessingResults);
if (!partProcessingResults.IsSuccessful)
{
break;
}
}
return(overallProcessingNumbers);
}
public ProcessLogSetResult ProcessLogSet()
{
_logger.LogInformation("Using temp folder `{tempDir}`", _config.TempDir);
using var logsExtractor = new TableauLogsExtractor(_config.LogSetLocation, _config.TempDir, _processingNotificationsCollector, _loggerFactory.CreateLogger <TableauLogsExtractor>());
if (!_pluginManager.IsValidPluginConfiguration(out var badPluginNames))
{
return(ProcessLogSetResult.Failed(FailureReasonMessageGenerator.BadPluginNamesSpecified(badPluginNames), ExitReason.IncorrectConfiguration));
}
var loadedPlugins = _pluginManager.CreatePlugins(_writerFactory, _processingNotificationsCollector).ToList();
var requiredLogTypes = _pluginManager.GetRequiredLogTypes().ToList();
var processedLogTypes = string.Join(", ", requiredLogTypes.OrderBy(name => name));
_logger.LogInformation("Based on requested plugins, the following log types will be processed: {processedLogTypes}", processedLogTypes);
var logProcessingStatistics = new Dictionary <LogType, ProcessLogTypeResult>();
var pluginsReceivedAnyData = new HashSet <string>();
ProcessLogTypeResult failedLogTypeResult = null;
foreach (var logType in requiredLogTypes)
{
_logger.LogInformation("Starting to process {logType} logs", logType);
var logTypeInfo = _logTypeDetails.GetInfoForLogType(logType);
var applicablePlugins = loadedPlugins
.Where(plugin => plugin.ConsumedLogTypes.Contains(logType))
.ToList();
try
{
var logTypeProcessingResult = ProcessLogType(logsExtractor.LogSetParts, logTypeInfo, applicablePlugins);
logProcessingStatistics.Add(logType, logTypeProcessingResult);
if (!logTypeProcessingResult.IsSuccessful)
{
failedLogTypeResult = logTypeProcessingResult;
break;
}
_logger.LogInformation("Done processing {logType} logs. {processingResult}", logType, logTypeProcessingResult);
if (logTypeProcessingResult.FilesProcessed > 0)
{
foreach (var plugin in applicablePlugins)
{
pluginsReceivedAnyData.Add(plugin.Name);
}
}
}
catch (Exception ex)
{
var unhandledExceptionMessage = $"Unhandled exception occurred while processing log type {logType}. Exception: {ex.Message}";
_logger.LogError(ex, unhandledExceptionMessage);
var fakeProcessingFileResult = new ProcessFileResult(0, unhandledExceptionMessage, ExitReason.UnclassifiedError);
var fakeProcessingTypeResults = new ProcessLogTypeResult();
fakeProcessingTypeResults.AddProcessingInfo(TimeSpan.Zero, 0, fakeProcessingFileResult);
failedLogTypeResult = fakeProcessingTypeResults;
break;
}
}
_logger.LogInformation("Telling all plugins to complete processing of any cached data");
var pluginsExecutionResults = _pluginManager.SendCompleteProcessingSignalToPlugins(failedLogTypeResult != null);
_logger.LogInformation("Completed reading log set and generating data");
var(errorMessage, existReason) = GetExitReasonAndErrorMessageIfApplicable(failedLogTypeResult, logProcessingStatistics);
var loadedPluginNames = loadedPlugins.Select(plugin => plugin.Name).ToHashSet();
return(new ProcessLogSetResult(
errorMessage,
existReason,
logsExtractor.LogSetSizeBytes,
logsExtractor.IsDirectory,
loadedPluginNames,
logProcessingStatistics,
pluginsExecutionResults,
pluginsReceivedAnyData));
}
