private static void GenerateWritersStatisticsSection(StringBuilder sb, WritersStatistics writersStatistics)
{
sb.AppendLine(GenerateTitle("Data Writers Statistics"));
if (writersStatistics == null)
{
sb.AppendLine("No writer statistics generated");
return;
}
var receivedAnything = writersStatistics.DataSets
.Where(pair => pair.Value.LinesPersisted > 0)
.OrderBy(pair => pair.Key.ToString())
.Select(pair => $"{pair.Key.ToString().PadRight(50, ' ')}: {pair.Value.LinesPersisted} lines persisted")
.ToList();
if (receivedAnything.Count > 0)
{
sb.AppendLine("Lines persisted per data set:");
sb.AppendLine(string.Join(Environment.NewLine, receivedAnything));
sb.AppendLine();
}
var receivedNulls = writersStatistics.DataSets
.Where(pair => pair.Value.NullLinesIgnored > 0)
.OrderBy(pair => pair.Key.ToString())
.Select(pair => $"{pair.Key.ToString().PadRight(40, ' ')}: {pair.Value.NullLinesIgnored} null lines ignored")
.ToList();
if (receivedNulls.Count > 0)
{
sb.AppendLine("Some writer(s) encountered null values. This is not a normal condition. Please contact LogShark owners for help.");
sb.AppendLine("Counts per data set:");
sb.AppendLine(string.Join(Environment.NewLine, receivedNulls));
}
}
public static HashSet <string> GetNonEmptyExtractNames(WritersStatistics writersStatistics)
{
return(writersStatistics.DataSets
.Where(pair => pair.Value.LinesPersisted > 0)
.Select(pair => pair.Key.Name + ".hyper")
.ToHashSet());
}
public WorkbookGeneratorResults CompleteWorkbooksWithResults(WritersStatistics writersStatistics)
{
_logger.LogInformation("Starting to generate workbooks with results...");
var availableTemplates = WorkbookGeneratorCommon.GenerateWorkbookTemplatesList(_config.WorkbookTemplatesDirectory, _config.CustomWorkbookTemplatesDirectory, _logger);
var applicableTemplates = WorkbookGeneratorCommon.SelectTemplatesApplicableToThisRun(availableTemplates, writersStatistics);
var nonEmptyExtractNames = WorkbookGeneratorCommon.GetNonEmptyExtractNames(writersStatistics);
var completedWorkbooks = applicableTemplates
.Select(template => CompleteWorkbook(template, WorkbookGeneratorCommon.HasAnyData(template, nonEmptyExtractNames)))
.ToList();
return(new WorkbookGeneratorResults(completedWorkbooks, availableTemplates));
}
public WorkbookGeneratorResults CompleteWorkbooksWithResults(WritersStatistics writersStatistics)
{
_logger.LogInformation("Starting to generate workbooks with results...");
var availableTemplates = WorkbookGeneratorCommon.GenerateWorkbookTemplatesList(_config.WorkbookTemplatesDirectory, _config.CustomWorkbookTemplatesDirectory, _logger);
var applicableTemplates = WorkbookGeneratorCommon.SelectTemplatesApplicableToThisRun(availableTemplates, writersStatistics);
var nonEmptyExtractNames = WorkbookGeneratorCommon.GetNonEmptyExtractNames(writersStatistics);
var results = new WorkbookGeneratorResults(new List <CompletedWorkbookInfo>(), availableTemplates);
foreach (var template in applicableTemplates)
{
var templateWorkbookPath = template.Path;
var hasAnyData = WorkbookGeneratorCommon.HasAnyData(template, nonEmptyExtractNames);
var baseWorkbookName = template.Name + _config.CustomWorkbookSuffix;
var finalWorkbookName = hasAnyData ? baseWorkbookName : $"{baseWorkbookName} [No Data]";
var destinationWorkbookPath = Path.Combine(_workbooksOutputDir, $"{finalWorkbookName}.twb");
try
{
CopyWorkbookToOutput(templateWorkbookPath, destinationWorkbookPath);
ReplaceConnectionInWorkbook(destinationWorkbookPath);
results.CompletedWorkbooks.Add(new CompletedWorkbookInfo(template.Name, finalWorkbookName, destinationWorkbookPath, hasAnyData));
}
catch (Exception ex)
{
var message = $"Exception occurred while generating workbook {template.Name}. Exception: {ex.Message}";
_logger.LogError(message);
results.CompletedWorkbooks.Add(new CompletedWorkbookInfo(template.Name, finalWorkbookName, workbookPath: null, hasAnyData: false, new WorkbookGeneratingException("", ex)));
}
}
return(results);
}
private static void AssertLoadAllResults(TestWriterFactory testWriterFactory, PluginManager pluginManager, WritersStatistics writersStatistics)
{
// As number of plugins can chance over time, we're simply verifying that two known plugins are loaded correctly
var loadedPlugins = pluginManager.GetPlugins();
loadedPlugins.Count().Should().BeGreaterThan(1);
loadedPlugins.FirstOrDefault(plugin => plugin.Name == "Apache").Should().NotBe(null);
loadedPlugins.FirstOrDefault(plugin => plugin.Name == "Filestore").Should().NotBe(null);
testWriterFactory.Writers.Count.Should().BeGreaterThan(1);
testWriterFactory.AssertAllWritersDisposedState(false);
writersStatistics.DataSets.Should().ContainKey(new DataSetInfo("Apache", "ApacheRequests"));
writersStatistics.DataSets.Should().ContainKey(new DataSetInfo("Filestore", "Filestore"));
}
public WorkbookGeneratorResults CompleteWorkbooksWithResults(WritersStatistics writersStatistics)
{
// TODO - do something here if we decide to use CSV files for workbooks
return(new WorkbookGeneratorResults(null, null));
}
/// <summary>
/// Filters the list of all available templates leaving only templates which can be used with the data generated writers in this run
/// </summary>
public static IEnumerable <PackagedWorkbookTemplateInfo> SelectTemplatesApplicableToThisRun(IEnumerable <PackagedWorkbookTemplateInfo> allTemplates, WritersStatistics writersStatistics)
{
var availableDataSets = writersStatistics.DataSets
.Select(kvp => kvp.Key.Name)
.Select(name => $"{name}.hyper") // PackagedWorkbookTemplateInfo has hyper file names in it
.ToHashSet();
return(allTemplates.Where(template => HaveAllDataSets(template, availableDataSets)));
}