private ProcessLogTypeResult ProcessZip(LogSetInfo logSetInfo, LogTypeInfo logTypeInfo, IList <IPlugin> plugins) { var processingStatistics = new ProcessLogTypeResult(); using var zip = ZipFile.Open(logSetInfo.Path, ZipArchiveMode.Read); foreach (var fileEntry in zip.Entries) { if (!logTypeInfo.FileBelongsToThisType(fileEntry.FullName)) { continue; } var fileNameWithPrefix = string.IsNullOrWhiteSpace(logSetInfo.Prefix) ? fileEntry.FullName : $"{logSetInfo.Prefix}/{fileEntry.FullName}"; _logger.LogInformation("Processing file {logFile}", fileNameWithPrefix); var fileTimer = Stopwatch.StartNew(); var processFileResult = ProcessZippedFile(fileEntry, fileNameWithPrefix, logTypeInfo, plugins); processingStatistics.AddProcessingInfo(fileTimer.Elapsed, fileEntry.Length, processFileResult); if (!processFileResult.IsSuccessful) { break; } LogFileProcessingResults(fileNameWithPrefix, fileEntry.Length, processFileResult, fileTimer.Elapsed); } return(processingStatistics); }
private ProcessLogTypeResult ProcessDir(LogSetInfo logSetInfo, LogTypeInfo logTypeInfo, IList <IPlugin> plugins) { var processingStatistics = new ProcessLogTypeResult(); foreach (var filePath in Directory.EnumerateFiles(logSetInfo.Path, "*", SearchOption.AllDirectories)) { var normalizedPath = filePath.NormalizePath(logSetInfo.Path); if (!logTypeInfo.FileBelongsToThisType(normalizedPath)) { continue; } _logger.LogInformation("Processing file {}", normalizedPath); var fileSizeBytes = new FileInfo(filePath).Length; var fileTimer = Stopwatch.StartNew(); var processFileResult = ProcessFile(filePath, normalizedPath, logTypeInfo, plugins); processingStatistics.AddProcessingInfo(fileTimer.Elapsed, fileSizeBytes, processFileResult); if (!processFileResult.IsSuccessful) { break; } LogFileProcessingResults(normalizedPath, fileSizeBytes, processFileResult, fileTimer.Elapsed); } return(processingStatistics); }
private (string, ExitReason) GetExitReasonAndErrorMessageIfApplicable(ProcessLogTypeResult failedProcessLogTypeResult, Dictionary <LogType, ProcessLogTypeResult> logProcessingStatistics) { if (failedProcessLogTypeResult != null) { return(failedProcessLogTypeResult.ErrorMessage, failedProcessLogTypeResult.ExitReason); } var processedAnyFiles = logProcessingStatistics.Any(kvp => kvp.Value.FilesProcessed > 0); return(processedAnyFiles ? (null, ExitReason.CompletedSuccessfully) : (FailureReasonMessageGenerator.NoTableauLogFilesFound(_config.RequestedPlugins, _config.OriginalLocation), ExitReason.LogSetDoesNotContainRelevantLogs)); }
private ProcessLogTypeResult ProcessLogType(IEnumerable <LogSetInfo> logSetParts, LogTypeInfo logTypeInfo, IList <IPlugin> plugins) { var overallProcessingNumbers = new ProcessLogTypeResult(); using var _ = _logger.BeginScope(logTypeInfo.LogType); foreach (var logSetInfo in logSetParts) { _logger.LogInformation("Starting to process log set part `{logSetPartPath}` for {logType} logs", logSetInfo.Path, logTypeInfo.LogType); var partProcessingResults = logSetInfo.IsZip ? ProcessZip(logSetInfo, logTypeInfo, plugins) : ProcessDir(logSetInfo, logTypeInfo, plugins); _logger.LogInformation("Completed processing `{logSetPartPath}` for {logType} logs. {partProcessingResults}", logSetInfo.Path, logTypeInfo.LogType, partProcessingResults); overallProcessingNumbers.AddNumbersFrom(partProcessingResults); if (!partProcessingResults.IsSuccessful) { break; } } return(overallProcessingNumbers); }
public ProcessLogSetResult ProcessLogSet() { _logger.LogInformation("Using temp folder `{tempDir}`", _config.TempDir); using var logsExtractor = new TableauLogsExtractor(_config.LogSetLocation, _config.TempDir, _processingNotificationsCollector, _loggerFactory.CreateLogger <TableauLogsExtractor>()); if (!_pluginManager.IsValidPluginConfiguration(out var badPluginNames)) { return(ProcessLogSetResult.Failed(FailureReasonMessageGenerator.BadPluginNamesSpecified(badPluginNames), ExitReason.IncorrectConfiguration)); } var loadedPlugins = _pluginManager.CreatePlugins(_writerFactory, _processingNotificationsCollector).ToList(); var requiredLogTypes = _pluginManager.GetRequiredLogTypes().ToList(); var processedLogTypes = string.Join(", ", requiredLogTypes.OrderBy(name => name)); _logger.LogInformation("Based on requested plugins, the following log types will be processed: {processedLogTypes}", processedLogTypes); var logProcessingStatistics = new Dictionary <LogType, ProcessLogTypeResult>(); var pluginsReceivedAnyData = new HashSet <string>(); ProcessLogTypeResult failedLogTypeResult = null; foreach (var logType in requiredLogTypes) { _logger.LogInformation("Starting to process {logType} logs", logType); var logTypeInfo = _logTypeDetails.GetInfoForLogType(logType); var applicablePlugins = loadedPlugins .Where(plugin => plugin.ConsumedLogTypes.Contains(logType)) .ToList(); try { var logTypeProcessingResult = ProcessLogType(logsExtractor.LogSetParts, logTypeInfo, applicablePlugins); logProcessingStatistics.Add(logType, logTypeProcessingResult); if (!logTypeProcessingResult.IsSuccessful) { failedLogTypeResult = logTypeProcessingResult; break; } _logger.LogInformation("Done processing {logType} logs. {processingResult}", logType, logTypeProcessingResult); if (logTypeProcessingResult.FilesProcessed > 0) { foreach (var plugin in applicablePlugins) { pluginsReceivedAnyData.Add(plugin.Name); } } } catch (Exception ex) { var unhandledExceptionMessage = $"Unhandled exception occurred while processing log type {logType}. Exception: {ex.Message}"; _logger.LogError(ex, unhandledExceptionMessage); var fakeProcessingFileResult = new ProcessFileResult(0, unhandledExceptionMessage, ExitReason.UnclassifiedError); var fakeProcessingTypeResults = new ProcessLogTypeResult(); fakeProcessingTypeResults.AddProcessingInfo(TimeSpan.Zero, 0, fakeProcessingFileResult); failedLogTypeResult = fakeProcessingTypeResults; break; } } _logger.LogInformation("Telling all plugins to complete processing of any cached data"); var pluginsExecutionResults = _pluginManager.SendCompleteProcessingSignalToPlugins(failedLogTypeResult != null); _logger.LogInformation("Completed reading log set and generating data"); var(errorMessage, existReason) = GetExitReasonAndErrorMessageIfApplicable(failedLogTypeResult, logProcessingStatistics); var loadedPluginNames = loadedPlugins.Select(plugin => plugin.Name).ToHashSet(); return(new ProcessLogSetResult( errorMessage, existReason, logsExtractor.LogSetSizeBytes, logsExtractor.IsDirectory, loadedPluginNames, logProcessingStatistics, pluginsExecutionResults, pluginsReceivedAnyData)); }